8205923: ZGC: Verification applies load barriers before verification

Reviewed-by: pliden, eosterlund

src/hotspot/share/gc/z/zCollectedHeap.cpp

@@ -233,11 +233,11 @@ GrowableArray<MemoryPool*> ZCollectedHeap::memory_pools() {
 }
 
 void ZCollectedHeap::object_iterate(ObjectClosure* cl) {
-  _heap.object_iterate(cl);
+  _heap.object_iterate(cl, true /* visit_referents */);
 }
 
 void ZCollectedHeap::safe_object_iterate(ObjectClosure* cl) {
-  _heap.object_iterate(cl);
+  _heap.object_iterate(cl, true /* visit_referents */);
 }
 
 HeapWord* ZCollectedHeap::block_start(const void* addr) const {

src/hotspot/share/gc/z/zHeap.cpp

@@ -503,10 +503,10 @@ void ZHeap::relocate() {
                                  used(), used_high(), used_low());
 }
 
-void ZHeap::object_iterate(ObjectClosure* cl) {
+void ZHeap::object_iterate(ObjectClosure* cl, bool visit_referents) {
   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
 
-  ZHeapIterator iter;
+  ZHeapIterator iter(visit_referents);
   iter.objects_do(cl);
 }
 
@@ -577,6 +577,6 @@ void ZHeap::verify() {
 
   {
     ZVerifyObjectClosure cl;
-    object_iterate(&cl);
+    object_iterate(&cl, false /* visit_referents */);
   }
 }

src/hotspot/share/gc/z/zHeap.hpp

@@ -152,7 +152,7 @@ public:
   void relocate();
 
   // Iteration
-  void object_iterate(ObjectClosure* cl);
+  void object_iterate(ObjectClosure* cl, bool visit_referents);
 
   // Serviceability
   void serviceability_initialize();

src/hotspot/share/gc/z/zHeapIterator.cpp

@@ -78,14 +78,28 @@ class ZHeapIteratorPushOopClosure : public BasicOopIterateClosure {
 private:
   ZHeapIterator* const _iter;
   const oop            _base;
+  const bool           _visit_referents;
 
 public:
   ZHeapIteratorPushOopClosure(ZHeapIterator* iter, oop base) :
       _iter(iter),
-      _base(base) {}
+      _base(base),
+      _visit_referents(iter->visit_referents()) {}
+
+  oop load_oop(oop* p) {
+    if (_visit_referents) {
+      return HeapAccess<ON_UNKNOWN_OOP_REF>::oop_load_at(_base, _base->field_offset(p));
+    } else {
+      return HeapAccess<>::oop_load(p);
+    }
+  }
+
+  virtual ReferenceIterationMode reference_iteration_mode() {
+    return _visit_referents ? DO_FIELDS : DO_FIELDS_EXCEPT_REFERENT;
+  }
 
   virtual void do_oop(oop* p) {
-    const oop obj = HeapAccess<ON_UNKNOWN_OOP_REF>::oop_load_at(_base, _base->field_offset(p));
+    const oop obj = load_oop(p);
     _iter->push(obj);
   }
 
@@ -100,9 +114,10 @@ public:
 #endif
 };
 
-ZHeapIterator::ZHeapIterator() :
+ZHeapIterator::ZHeapIterator(bool visit_referents) :
     _visit_stack(),
-    _visit_map() {}
+    _visit_map(),
+    _visit_referents(visit_referents) {}
 
 ZHeapIterator::~ZHeapIterator() {
   ZVisitMapIterator iter(&_visit_map);
@@ -163,6 +178,10 @@ void ZHeapIterator::drain(ObjectClosure* cl) {
   }
 }
 
+bool ZHeapIterator::visit_referents() const {
+  return _visit_referents;
+}
+
 void ZHeapIterator::objects_do(ObjectClosure* cl) {
   ZHeapIteratorRootOopClosure root_cl(this, cl);
   ZRootsIterator roots;

src/hotspot/share/gc/z/zHeapIterator.hpp

@@ -42,6 +42,7 @@ private:
 
   ZVisitStack _visit_stack;
   ZVisitMap   _visit_map;
+  const bool  _visit_referents;
 
   size_t object_index_max() const;
   size_t object_index(oop obj) const;
@@ -50,8 +51,10 @@ private:
   void push(oop obj);
   void drain(ObjectClosure* cl);
 
+  bool visit_referents() const;
+
 public:
-  ZHeapIterator();
+  ZHeapIterator(bool visit_referents);
   ~ZHeapIterator();
 
   void objects_do(ObjectClosure* cl);

src/hotspot/share/gc/z/zOopClosures.cpp

@@ -40,13 +40,17 @@ static void z_verify_loaded_object(const oop* p, const oop obj) {
             p2i(obj), p2i(p));
 }
 
-ZVerifyHeapOopClosure::ZVerifyHeapOopClosure(oop base)
-    : _base(base) {}
+OopIterateClosure::ReferenceIterationMode ZVerifyHeapOopClosure::reference_iteration_mode() {
+  // Don't visit the j.l.Reference.referents for this verification closure,
+  // since they are cleaned concurrently after ZHeap::mark_end(), and can
+  // therefore not be verified at this point.
+  return DO_FIELDS_EXCEPT_REFERENT;
+}
 
 void ZVerifyHeapOopClosure::do_oop(oop* p) {
   guarantee(ZHeap::heap()->is_in((uintptr_t)p), "oop* " PTR_FORMAT " not in heap", p2i(p));
 
-  const oop obj = HeapAccess<ON_UNKNOWN_OOP_REF>::oop_load_at(_base, _base->field_offset(p));
+  const oop obj = RawAccess<>::oop_load(p);
   z_verify_loaded_object(p, obj);
 }
 
@@ -54,10 +58,16 @@ void ZVerifyHeapOopClosure::do_oop(narrowOop* p) {
   ShouldNotReachHere();
 }
 
+ZVerifyRootOopClosure::ZVerifyRootOopClosure() {
+  // This closure should only be used from ZHeap::mark_end(),
+  // when all roots should have been fixed by the fixup_partial_loads().
+  guarantee(ZGlobalPhase == ZPhaseMarkCompleted, "Invalid phase");
+}
+
 void ZVerifyRootOopClosure::do_oop(oop* p) {
   guarantee(!ZHeap::heap()->is_in((uintptr_t)p), "oop* " PTR_FORMAT " in heap", p2i(p));
 
-  const oop obj = NativeAccess<>::oop_load(p);
+  const oop obj = RawAccess<>::oop_load(p);
   z_verify_loaded_object(p, obj);
 }
 
@@ -66,6 +76,6 @@ void ZVerifyRootOopClosure::do_oop(narrowOop* p) {
 }
 
 void ZVerifyObjectClosure::do_object(oop o) {
-  ZVerifyHeapOopClosure cl(o);
+  ZVerifyHeapOopClosure cl;
   o->oop_iterate(&cl);
 }

src/hotspot/share/gc/z/zOopClosures.hpp

@@ -83,11 +83,8 @@ public:
 };
 
 class ZVerifyHeapOopClosure : public BasicOopIterateClosure {
-private:
-  const oop _base;
-
 public:
-  ZVerifyHeapOopClosure(oop base);
+  virtual ReferenceIterationMode reference_iteration_mode();
 
   virtual void do_oop(oop* p);
   virtual void do_oop(narrowOop* p);
@@ -102,6 +99,8 @@ public:
 
 class ZVerifyRootOopClosure : public OopClosure {
 public:
+  ZVerifyRootOopClosure();
+
   virtual void do_oop(oop* p);
   virtual void do_oop(narrowOop* p);
 };