8367972: ZGC: Reduce ZBarrierSet includes

Reviewed-by: stefank, eosterlund

src/hotspot/share/gc/z/zBarrierSet.cpp

@@ -21,6 +21,8 @@
  * questions.
  */
 
+#include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zBarrier.inline.hpp"
 #include "gc/z/zBarrierSet.hpp"
 #include "gc/z/zBarrierSetAssembler.hpp"
 #include "gc/z/zBarrierSetNMethod.hpp"
@@ -30,6 +32,7 @@
 #include "gc/z/zHeap.inline.hpp"
 #include "gc/z/zStackWatermark.hpp"
 #include "gc/z/zThreadLocalData.hpp"
+#include "runtime/atomicAccess.hpp"
 #include "runtime/deoptimization.hpp"
 #include "runtime/frame.inline.hpp"
 #include "runtime/javaThread.hpp"
@@ -46,6 +49,96 @@
 class ZBarrierSetC1;
 class ZBarrierSetC2;
 
+class ZColorStoreGoodOopClosure : public BasicOopIterateClosure {
+public:
+  virtual void do_oop(oop* p_) {
+    volatile zpointer* const p = (volatile zpointer*)p_;
+    const zpointer ptr = ZBarrier::load_atomic(p);
+    const zaddress addr = ZPointer::uncolor(ptr);
+    AtomicAccess::store(p, ZAddress::store_good(addr));
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+};
+
+class ZLoadBarrierOopClosure : public BasicOopIterateClosure {
+public:
+  virtual void do_oop(oop* p) {
+    ZBarrier::load_barrier_on_oop_field((zpointer*)p);
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+};
+
+void ZBarrierSet::load_barrier_all(oop src, size_t size) {
+  check_is_valid_zaddress(src);
+
+  ZLoadBarrierOopClosure cl;
+  ZIterator::oop_iterate(src, &cl);
+}
+
+void ZBarrierSet::color_store_good_all(oop dst, size_t size) {
+  check_is_valid_zaddress(dst);
+  assert(dst->is_typeArray() || ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
+
+  ZColorStoreGoodOopClosure cl_sg;
+  ZIterator::oop_iterate(dst, &cl_sg);
+}
+
+zaddress ZBarrierSet::load_barrier_on_oop_field_preloaded(volatile zpointer* p, zpointer o) {
+  return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
+}
+
+zaddress ZBarrierSet::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o) {
+  return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
+}
+
+zaddress ZBarrierSet::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o) {
+  return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
+}
+
+zaddress ZBarrierSet::load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o) {
+  return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
+}
+
+zaddress ZBarrierSet::load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o) {
+  return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
+}
+
+void ZBarrierSet::store_barrier_on_heap_oop_field(volatile zpointer* p, bool heal) {
+  ZBarrier::store_barrier_on_heap_oop_field(p, heal);
+}
+
+void ZBarrierSet::no_keep_alive_store_barrier_on_heap_oop_field(volatile zpointer* p) {
+  ZBarrier::no_keep_alive_store_barrier_on_heap_oop_field(p);
+}
+
+void ZBarrierSet::store_barrier_on_native_oop_field(volatile zpointer* p, bool heal) {
+  ZBarrier::store_barrier_on_native_oop_field(p, heal);
+}
+
+zaddress ZBarrierSet::load_barrier_on_oop_field(volatile zpointer* p) {
+  return ZBarrier::load_barrier_on_oop_field(p);
+}
+
+void ZBarrierSet::clone_obj_array(objArrayOop src_obj, objArrayOop dst_obj) {
+  volatile zpointer* src = (volatile zpointer*)src_obj->base();
+  volatile zpointer* dst = (volatile zpointer*)dst_obj->base();
+  const int length = src_obj->length();
+
+  for (const volatile zpointer* const end = src + length; src < end; src++, dst++) {
+    zaddress elem = ZBarrier::load_barrier_on_oop_field(src);
+    // We avoid healing here because the store below colors the pointer store good,
+    // hence avoiding the cost of a CAS.
+    ZBarrier::store_barrier_on_heap_oop_field(dst, false /* heal */);
+    AtomicAccess::store(dst, ZAddress::store_good(elem));
+  }
+}
+
 ZBarrierSet::ZBarrierSet()
   : BarrierSet(make_barrier_set_assembler<ZBarrierSetAssembler>(),
                make_barrier_set_c1<ZBarrierSetC1>(),
@@ -153,20 +246,6 @@ void ZBarrierSet::on_slowpath_allocation_exit(JavaThread* thread, oop new_obj) {
   deoptimize_allocation(thread);
 }
 
-void ZBarrierSet::clone_obj_array(objArrayOop src_obj, objArrayOop dst_obj) {
-  volatile zpointer* src = (volatile zpointer*)src_obj->base();
-  volatile zpointer* dst = (volatile zpointer*)dst_obj->base();
-  const int length = src_obj->length();
-
-  for (const volatile zpointer* const end = src + length; src < end; src++, dst++) {
-    zaddress elem = ZBarrier::load_barrier_on_oop_field(src);
-    // We avoid healing here because the store below colors the pointer store good,
-    // hence avoiding the cost of a CAS.
-    ZBarrier::store_barrier_on_heap_oop_field(dst, false /* heal */);
-    AtomicAccess::store(dst, ZAddress::store_good(elem));
-  }
-}
-
 void ZBarrierSet::print_on(outputStream* st) const {
   st->print_cr("ZBarrierSet");
 }

src/hotspot/share/gc/z/zBarrierSet.hpp

@@ -33,14 +33,29 @@ class ZBarrierSet : public BarrierSet {
 private:
   static zpointer store_good(oop obj);
 
+  static void load_barrier_all(oop src, size_t size);
+  static void color_store_good_all(oop dst, size_t size);
+
+  static zaddress load_barrier_on_oop_field_preloaded(volatile zpointer* p, zpointer o);
+  static zaddress no_keep_alive_load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o);
+  static zaddress no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o);
+  static zaddress load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o);
+  static zaddress load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o);
+
+  static void store_barrier_on_heap_oop_field(volatile zpointer* p, bool heal);
+  static void no_keep_alive_store_barrier_on_heap_oop_field(volatile zpointer* p);
+  static void store_barrier_on_native_oop_field(volatile zpointer* p, bool heal);
+
+  static zaddress load_barrier_on_oop_field(volatile zpointer* p);
+
+  static void clone_obj_array(objArrayOop src, objArrayOop dst);
+
 public:
   ZBarrierSet();
 
   static ZBarrierSetAssembler* assembler();
   static bool barrier_needed(DecoratorSet decorators, BasicType type);
 
-  static void clone_obj_array(objArrayOop src, objArrayOop dst);
-
   virtual void on_thread_create(Thread* thread);
   virtual void on_thread_destroy(Thread* thread);
   virtual void on_thread_attach(Thread* thread);

src/hotspot/share/gc/z/zBarrierSet.inline.hpp

@@ -28,10 +28,9 @@
 
 #include "gc/shared/accessBarrierSupport.inline.hpp"
 #include "gc/z/zAddress.inline.hpp"
-#include "gc/z/zBarrier.inline.hpp"
-#include "gc/z/zIterator.inline.hpp"
+#include "gc/z/zHeap.hpp"
 #include "gc/z/zNMethod.hpp"
-#include "memory/iterator.inline.hpp"
+#include "oops/objArrayOop.hpp"
 #include "utilities/debug.hpp"
 
 template <DecoratorSet decorators, typename BarrierSetT>
@@ -68,21 +67,21 @@ inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrie
   if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
     if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
       // Load barriers on strong oop refs don't keep objects alive
-      return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
+      return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o);
     } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
-      return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
+      return ZBarrierSet::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
     } else {
       assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
-      return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
+      return ZBarrierSet::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
     }
   } else {
     if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
-      return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
+      return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o);
     } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
-      return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
+      return ZBarrierSet::load_barrier_on_weak_oop_field_preloaded(p, o);
     } else {
       assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
-      return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
+      return ZBarrierSet::load_barrier_on_phantom_oop_field_preloaded(p, o);
     }
   }
 }
@@ -97,21 +96,21 @@ inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrie
   if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
     if (decorators_known_strength & ON_STRONG_OOP_REF) {
       // Load barriers on strong oop refs don't keep objects alive
-      return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
+      return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o);
     } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
-      return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
+      return ZBarrierSet::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
     } else {
       assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
-      return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
+      return ZBarrierSet::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
     }
   } else {
     if (decorators_known_strength & ON_STRONG_OOP_REF) {
-      return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
+      return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o);
     } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
-      return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
+      return ZBarrierSet::load_barrier_on_weak_oop_field_preloaded(p, o);
     } else {
       assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
-      return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
+      return ZBarrierSet::load_barrier_on_phantom_oop_field_preloaded(p, o);
     }
   }
 }
@@ -126,7 +125,7 @@ inline zpointer ZBarrierSet::store_good(oop obj) {
 template <DecoratorSet decorators, typename BarrierSetT>
 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_with_healing(zpointer* p) {
   if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
-    ZBarrier::store_barrier_on_heap_oop_field(p, true /* heal */);
+    ZBarrierSet::store_barrier_on_heap_oop_field(p, true /* heal */);
   } else {
     assert(false, "Should not be used on uninitialized memory");
   }
@@ -135,21 +134,21 @@ inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_h
 template <DecoratorSet decorators, typename BarrierSetT>
 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_without_healing(zpointer* p) {
   if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
-    ZBarrier::store_barrier_on_heap_oop_field(p, false /* heal */);
+    ZBarrierSet::store_barrier_on_heap_oop_field(p, false /* heal */);
   }
 }
 
 template <DecoratorSet decorators, typename BarrierSetT>
 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::no_keep_alive_store_barrier_heap(zpointer* p) {
   if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
-    ZBarrier::no_keep_alive_store_barrier_on_heap_oop_field(p);
+    ZBarrierSet::no_keep_alive_store_barrier_on_heap_oop_field(p);
   }
 }
 
 template <DecoratorSet decorators, typename BarrierSetT>
 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_with_healing(zpointer* p) {
   if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
-    ZBarrier::store_barrier_on_native_oop_field(p, true /* heal */);
+    ZBarrierSet::store_barrier_on_native_oop_field(p, true /* heal */);
   } else {
     assert(false, "Should not be used on uninitialized memory");
   }
@@ -158,7 +157,7 @@ inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_n
 template <DecoratorSet decorators, typename BarrierSetT>
 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_without_healing(zpointer* p) {
   if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
-    ZBarrier::store_barrier_on_native_oop_field(p, false /* heal */);
+    ZBarrierSet::store_barrier_on_native_oop_field(p, false /* heal */);
   }
 }
 
@@ -325,7 +324,7 @@ template <DecoratorSet decorators, typename BarrierSetT>
 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_barriers(zpointer* dst, zpointer* src) {
   store_barrier_heap_without_healing(dst);
 
-  return ZBarrier::load_barrier_on_oop_field(src);
+  return ZBarrierSet::load_barrier_on_oop_field(src);
 }
 
 template <DecoratorSet decorators, typename BarrierSetT>
@@ -402,31 +401,6 @@ inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_i
   return oop_arraycopy_in_heap_no_check_cast(dst, src, length);
 }
 
-class ZColorStoreGoodOopClosure : public BasicOopIterateClosure {
-public:
-  virtual void do_oop(oop* p_) {
-    volatile zpointer* const p = (volatile zpointer*)p_;
-    const zpointer ptr = ZBarrier::load_atomic(p);
-    const zaddress addr = ZPointer::uncolor(ptr);
-    AtomicAccess::store(p, ZAddress::store_good(addr));
-  }
-
-  virtual void do_oop(narrowOop* p) {
-    ShouldNotReachHere();
-  }
-};
-
-class ZLoadBarrierOopClosure : public BasicOopIterateClosure {
-public:
-  virtual void do_oop(oop* p) {
-    ZBarrier::load_barrier_on_oop_field((zpointer*)p);
-  }
-
-  virtual void do_oop(narrowOop* p) {
-    ShouldNotReachHere();
-  }
-};
-
 template <DecoratorSet decorators, typename BarrierSetT>
 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
   check_is_valid_zaddress(src);
@@ -443,17 +417,13 @@ inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(o
   }
 
   // Fix the oops
-  ZLoadBarrierOopClosure cl;
-  ZIterator::oop_iterate(src, &cl);
+  ZBarrierSet::load_barrier_all(src, size);
 
   // Clone the object
   Raw::clone_in_heap(src, dst, size);
 
-  assert(dst->is_typeArray() || ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
-
   // Color store good before handing out
-  ZColorStoreGoodOopClosure cl_sg;
-  ZIterator::oop_iterate(dst, &cl_sg);
+  ZBarrierSet::color_store_good_all(dst, size);
 }
 
 //

src/hotspot/share/gc/z/zObjArrayAllocator.cpp

@@ -21,6 +21,7 @@
  * questions.
  */
 
+#include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zObjArrayAllocator.hpp"
 #include "gc/z/zThreadLocalData.hpp"
 #include "gc/z/zUtils.inline.hpp"

src/hotspot/share/gc/z/zPhysicalMemoryManager.cpp

@@ -25,6 +25,7 @@
 #include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zArray.inline.hpp"
 #include "gc/z/zGlobals.hpp"
+#include "gc/z/zGranuleMap.inline.hpp"
 #include "gc/z/zLargePages.inline.hpp"
 #include "gc/z/zList.inline.hpp"
 #include "gc/z/zNMT.hpp"

src/hotspot/share/gc/z/zRangeRegistry.inline.hpp

@@ -27,6 +27,7 @@
 #include "gc/z/zRangeRegistry.hpp"
 
 #include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zArray.inline.hpp"
 #include "gc/z/zList.inline.hpp"
 #include "gc/z/zLock.inline.hpp"
 

src/hotspot/share/precompiled/precompiled.hpp

@@ -45,7 +45,6 @@
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #endif
 #if INCLUDE_ZGC
-#include "gc/z/zBarrier.inline.hpp"
 #include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zHeap.inline.hpp"
 #endif

src/hotspot/share/prims/whitebox.cpp

@@ -120,6 +120,7 @@
 #endif // INCLUDE_SERIALGC
 #if INCLUDE_ZGC
 #include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zHeap.inline.hpp"
 #endif // INCLUDE_ZGC
 #if INCLUDE_JVMCI
 #include "jvmci/jvmciEnv.hpp"

src/hotspot/share/runtime/stackValue.cpp

@@ -30,9 +30,6 @@
 #include "runtime/globals.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/stackValue.hpp"
-#if INCLUDE_ZGC
-#include "gc/z/zBarrier.inline.hpp"
-#endif
 #if INCLUDE_SHENANDOAHGC
 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
 #endif

test/hotspot/gtest/runtime/test_os_windows.cpp

@@ -27,7 +27,7 @@
 #include "logging/log.hpp"
 #include "runtime/flags/flagSetting.hpp"
 #include "runtime/globals_extension.hpp"
-#include "runtime/os.hpp"
+#include "runtime/os.inline.hpp"
 #include "concurrentTestRunner.inline.hpp"
 #include "unittest.hpp"