8372738: ZGC: C2 allocation reloc promotion deopt race

Reviewed-by: aboldtch, stefank

src/hotspot/share/gc/z/zBarrierSet.cpp

@@ -217,11 +217,10 @@ static void deoptimize_allocation(JavaThread* thread) {
 
 void ZBarrierSet::on_slowpath_allocation_exit(JavaThread* thread, oop new_obj) {
   const ZPage* const page = ZHeap::heap()->page(to_zaddress(new_obj));
-  const ZPageAge age = page->age();
-  if (age == ZPageAge::old) {
+  if (!page->allows_raw_null()) {
     // We promised C2 that its allocations would end up in young gen. This object
-    // breaks that promise. Take a few steps in the interpreter instead, which has
-    // no such assumptions about where an object resides.
+    // is too old to guarantee that. Take a few steps in the interpreter instead,
+    // which does not elide barriers based on the age of an object.
     deoptimize_allocation(thread);
   }
 }

src/hotspot/share/gc/z/zGeneration.cpp

@@ -190,7 +190,8 @@ void ZGeneration::flip_age_pages(const ZRelocationSetSelector* selector) {
   ZRendezvousHandshakeClosure cl;
   Handshake::execute(&cl);
 
-  _relocate.barrier_flip_promoted_pages(_relocation_set.flip_promoted_pages());
+  _relocate.barrier_promoted_pages(_relocation_set.flip_promoted_pages(),
+                                   _relocation_set.relocate_promoted_pages());
 }
 
 static double fragmentation_limit(ZGenerationId generation) {

src/hotspot/share/gc/z/zPage.cpp

@@ -41,7 +41,8 @@ ZPage::ZPage(ZPageType type, ZPageAge age, const ZVirtualMemory& vmem, ZMultiPar
     _top(to_zoffset_end(start())),
     _livemap(object_max_count()),
     _remembered_set(),
-    _multi_partition_tracker(multi_partition_tracker) {
+    _multi_partition_tracker(multi_partition_tracker),
+    _relocate_promoted(false) {
   assert(!_virtual.is_null(), "Should not be null");
   assert((_type == ZPageType::small && size() == ZPageSizeSmall) ||
          (_type == ZPageType::medium && ZPageSizeMediumMin <= size() && size() <= ZPageSizeMediumMax) ||
@@ -70,6 +71,14 @@ ZPage* ZPage::clone_for_promotion() const {
   return page;
 }
 
+bool ZPage::allows_raw_null() const {
+  return is_young() && !AtomicAccess::load(&_relocate_promoted);
+}
+
+void ZPage::set_is_relocate_promoted() {
+  AtomicAccess::store(&_relocate_promoted, true);
+}
+
 ZGeneration* ZPage::generation() {
   return ZGeneration::generation(_generation_id);
 }

src/hotspot/share/gc/z/zPage.hpp

@@ -52,6 +52,7 @@ private:
   ZLiveMap                      _livemap;
   ZRememberedSet                _remembered_set;
   ZMultiPartitionTracker* const _multi_partition_tracker;
+  volatile bool                 _relocate_promoted;
 
   const char* type_to_string() const;
 
@@ -103,6 +104,9 @@ public:
 
   ZPageAge age() const;
 
+  bool allows_raw_null() const;
+  void set_is_relocate_promoted();
+
   uint32_t seqnum() const;
   bool is_allocating() const;
   bool is_relocatable() const;

src/hotspot/share/gc/z/zRelocate.cpp

@@ -1366,27 +1366,35 @@ public:
 
 class ZPromoteBarrierTask : public ZTask {
 private:
-  ZArrayParallelIterator<ZPage*> _iter;
+  ZArrayParallelIterator<ZPage*> _flip_promoted_iter;
+  ZArrayParallelIterator<ZPage*> _relocate_promoted_iter;
 
 public:
-  ZPromoteBarrierTask(const ZArray<ZPage*>* pages)
+  ZPromoteBarrierTask(const ZArray<ZPage*>* flip_promoted_pages,
+                      const ZArray<ZPage*>* relocate_promoted_pages)
     : ZTask("ZPromoteBarrierTask"),
-      _iter(pages) {}
+      _flip_promoted_iter(flip_promoted_pages),
+      _relocate_promoted_iter(relocate_promoted_pages) {}
 
   virtual void work() {
     SuspendibleThreadSetJoiner sts_joiner;
 
-    for (ZPage* page; _iter.next(&page);) {
-      // When promoting an object (and before relocate start), we must ensure that all
-      // contained zpointers are store good. The marking code ensures that for non-null
-      // pointers, but null pointers are ignored. This code ensures that even null pointers
-      // are made store good, for the promoted objects.
-      page->object_iterate([&](oop obj) {
-        ZIterator::basic_oop_iterate_safe(obj, ZBarrier::promote_barrier_on_young_oop_field);
-      });
+    auto promote_barriers = [&](ZArrayParallelIterator<ZPage*>* iter) {
+      for (ZPage* page; iter->next(&page);) {
+        // When promoting an object (and before relocate start), we must ensure that all
+        // contained zpointers are store good. The marking code ensures that for non-null
+        // pointers, but null pointers are ignored. This code ensures that even null pointers
+        // are made store good, for the promoted objects.
+        page->object_iterate([&](oop obj) {
+          ZIterator::basic_oop_iterate_safe(obj, ZBarrier::promote_barrier_on_young_oop_field);
+        });
 
-      SuspendibleThreadSet::yield();
-    }
+        SuspendibleThreadSet::yield();
+      }
+    };
+
+    promote_barriers(&_flip_promoted_iter);
+    promote_barriers(&_relocate_promoted_iter);
   }
 };
 
@@ -1395,8 +1403,9 @@ void ZRelocate::flip_age_pages(const ZArray<ZPage*>* pages) {
   workers()->run(&flip_age_task);
 }
 
-void ZRelocate::barrier_flip_promoted_pages(const ZArray<ZPage*>* pages) {
-  ZPromoteBarrierTask promote_barrier_task(pages);
+void ZRelocate::barrier_promoted_pages(const ZArray<ZPage*>* flip_promoted_pages,
+                                       const ZArray<ZPage*>* relocate_promoted_pages) {
+  ZPromoteBarrierTask promote_barrier_task(flip_promoted_pages, relocate_promoted_pages);
   workers()->run(&promote_barrier_task);
 }
 

src/hotspot/share/gc/z/zRelocate.hpp

@@ -119,7 +119,8 @@ public:
   void relocate(ZRelocationSet* relocation_set);
 
   void flip_age_pages(const ZArray<ZPage*>* pages);
-  void barrier_flip_promoted_pages(const ZArray<ZPage*>* pages);
+  void barrier_promoted_pages(const ZArray<ZPage*>* flip_promoted_pages,
+                              const ZArray<ZPage*>* relocate_promoted_pages);
 
   void synchronize();
   void desynchronize();

src/hotspot/share/gc/z/zRelocationSet.cpp

@@ -38,6 +38,7 @@
 
 class ZRelocationSetInstallTask : public ZTask {
 private:
+  ZRelocationSet*                _relocation_set;
   ZForwardingAllocator* const    _allocator;
   ZForwarding**                  _forwardings;
   const size_t                   _nforwardings;
@@ -54,16 +55,6 @@ private:
     page->log_msg(" (relocation selected)");
 
     _forwardings[index] = forwarding;
-
-    if (forwarding->is_promotion()) {
-      // Before promoting an object (and before relocate start), we must ensure that all
-      // contained zpointers are store good. The marking code ensures that for non-null
-      // pointers, but null pointers are ignored. This code ensures that even null pointers
-      // are made store good, for the promoted objects.
-      page->object_iterate([&](oop obj) {
-        ZIterator::basic_oop_iterate_safe(obj, ZBarrier::promote_barrier_on_young_oop_field);
-      });
-    }
   }
 
   void install_small(ZForwarding* forwarding, size_t index) {
@@ -78,10 +69,18 @@ private:
     return ZRelocate::compute_to_age(page->age());
   }
 
+  void track_if_promoted(ZPage* page, ZForwarding* forwarding, ZArray<ZPage*>& relocate_promoted) {
+    if (forwarding->is_promotion()) {
+      page->set_is_relocate_promoted();
+      relocate_promoted.append(page);
+    }
+  }
+
 public:
-  ZRelocationSetInstallTask(ZForwardingAllocator* allocator, const ZRelocationSetSelector* selector)
+  ZRelocationSetInstallTask(ZRelocationSet* relocation_set, const ZRelocationSetSelector* selector)
     : ZTask("ZRelocationSetInstallTask"),
-      _allocator(allocator),
+      _relocation_set(relocation_set),
+      _allocator(&relocation_set->_allocator),
       _forwardings(nullptr),
       _nforwardings((size_t)selector->selected_small()->length() + (size_t)selector->selected_medium()->length()),
       _small(selector->selected_small()),
@@ -108,11 +107,14 @@ public:
     // Join the STS to block out VMThreads while running promote_barrier_on_young_oop_field
     SuspendibleThreadSetJoiner sts_joiner;
 
+    ZArray<ZPage*> relocate_promoted;
+
     // Allocate and install forwardings for small pages
     for (size_t page_index; _small_iter.next_index(&page_index);) {
       ZPage* page = _small->at(int(page_index));
       ZForwarding* const forwarding = ZForwarding::alloc(_allocator, page, to_age(page));
       install_small(forwarding, (size_t)_medium->length() + page_index);
+      track_if_promoted(page, forwarding, relocate_promoted);
 
       SuspendibleThreadSet::yield();
     }
@@ -122,9 +124,12 @@ public:
       ZPage* page = _medium->at(int(page_index));
       ZForwarding* const forwarding = ZForwarding::alloc(_allocator, page, to_age(page));
       install_medium(forwarding, page_index);
+      track_if_promoted(page, forwarding, relocate_promoted);
 
       SuspendibleThreadSet::yield();
     }
+
+    _relocation_set->register_relocate_promoted(relocate_promoted);
   }
 
   ZForwarding** forwardings() const {
@@ -143,6 +148,7 @@ ZRelocationSet::ZRelocationSet(ZGeneration* generation)
     _nforwardings(0),
     _promotion_lock(),
     _flip_promoted_pages(),
+    _relocate_promoted_pages(),
     _in_place_relocate_promoted_pages() {}
 
 ZWorkers* ZRelocationSet::workers() const {
@@ -157,9 +163,13 @@ ZArray<ZPage*>* ZRelocationSet::flip_promoted_pages() {
   return &_flip_promoted_pages;
 }
 
+ZArray<ZPage*>* ZRelocationSet::relocate_promoted_pages() {
+  return &_relocate_promoted_pages;
+}
+
 void ZRelocationSet::install(const ZRelocationSetSelector* selector) {
   // Install relocation set
-  ZRelocationSetInstallTask task(&_allocator, selector);
+  ZRelocationSetInstallTask task(this, selector);
   workers()->run(&task);
 
   _forwardings = task.forwardings();
@@ -189,6 +199,7 @@ void ZRelocationSet::reset(ZPageAllocator* page_allocator) {
 
   destroy_and_clear(page_allocator, &_in_place_relocate_promoted_pages);
   destroy_and_clear(page_allocator, &_flip_promoted_pages);
+  _relocate_promoted_pages.clear();
 }
 
 void ZRelocationSet::register_flip_promoted(const ZArray<ZPage*>& pages) {
@@ -199,6 +210,18 @@ void ZRelocationSet::register_flip_promoted(const ZArray<ZPage*>& pages) {
   }
 }
 
+void ZRelocationSet::register_relocate_promoted(const ZArray<ZPage*>& pages) {
+  if (pages.is_empty()) {
+    return;
+  }
+
+  ZLocker<ZLock> locker(&_promotion_lock);
+  for (ZPage* const page : pages) {
+    assert(!_relocate_promoted_pages.contains(page), "no duplicates allowed");
+    _relocate_promoted_pages.append(page);
+  }
+}
+
 void ZRelocationSet::register_in_place_relocate_promoted(ZPage* page) {
   ZLocker<ZLock> locker(&_promotion_lock);
   assert(!_in_place_relocate_promoted_pages.contains(page), "no duplicates allowed");

src/hotspot/share/gc/z/zRelocationSet.hpp

@@ -37,6 +37,7 @@ class ZWorkers;
 
 class ZRelocationSet {
   template <bool> friend class ZRelocationSetIteratorImpl;
+  friend class ZRelocationSetInstallTask;
 
 private:
   ZGeneration*         _generation;
@@ -45,6 +46,7 @@ private:
   size_t               _nforwardings;
   ZLock                _promotion_lock;
   ZArray<ZPage*>       _flip_promoted_pages;
+  ZArray<ZPage*>       _relocate_promoted_pages;
   ZArray<ZPage*>       _in_place_relocate_promoted_pages;
 
   ZWorkers* workers() const;
@@ -58,8 +60,10 @@ public:
   void reset(ZPageAllocator* page_allocator);
   ZGeneration* generation() const;
   ZArray<ZPage*>* flip_promoted_pages();
+  ZArray<ZPage*>* relocate_promoted_pages();
 
   void register_flip_promoted(const ZArray<ZPage*>& pages);
+  void register_relocate_promoted(const ZArray<ZPage*>& pages);
   void register_in_place_relocate_promoted(ZPage* page);
 };