Avoiding calling reset_age and set_update_watermark in fast alloc path

src/hotspot/share/gc/shenandoah/shenandoahAllocator.cpp

@@ -185,6 +185,10 @@ HeapWord* ShenandoahAllocator<ALLOC_PARTITION>::attempt_allocation_from_free_set
   if (r != nullptr) {
     bool ready_for_retire = false;
     HeapWord *obj = allocate_in<false>(r, false, req, in_new_region, ready_for_retire);
+    r->reset_age();
+    if (ALLOC_PARTITION != ShenandoahFreeSetPartitionId::Mutator) {
+      r->set_update_watermark(r->top<false>());
+    }
     assert(obj != nullptr, "Should always succeed.");
 
     _free_set->partitions()->increase_used(ALLOC_PARTITION, (req.actual_size() + req.waste()) * HeapWordSize);
@@ -258,17 +262,6 @@ HeapWord* ShenandoahAllocator<ALLOC_PARTITION>::allocate_in(ShenandoahHeapRegion
       _alloc_partition_name, actual_size * HeapWordSize, region->index(), req.size() * HeapWordSize, is_alloc_region ? "true" : "false", region->free());
     req.set_actual_size(actual_size);
     in_new_region = obj == region->bottom(); // is in new region when the allocated object is at the bottom of the region.
-    if (ALLOC_PARTITION != ShenandoahFreeSetPartitionId::Mutator) {
-      // For GC allocations, we advance update_watermark because the objects relocated into this memory during
-      // evacuation are not updated during evacuation.  For both young and old regions r, it is essential that all
-      // PLABs be made parsable at the end of evacuation.  This is enabled by retiring all plabs at end of evacuation.
-      if (IS_SHARED_ALLOC_REGION) {
-        region->concurrent_set_update_watermark(region->top<true>());
-      } else {
-        region->set_update_watermark(region->top<false>());
-      }
-    }
-
     if (!IS_SHARED_ALLOC_REGION && region->free_words() < PLAB::min_size()) {
       ready_for_retire = true;
     }
@@ -297,6 +290,10 @@ int ShenandoahAllocator<ALLOC_PARTITION>::replenish_alloc_regions(ShenandoahAllo
     if (region == nullptr || free_bytes / HeapWordSize < PLAB::min_size()) {
       if (region != nullptr) {
         region->unset_active_alloc_region();
+        if (ALLOC_PARTITION != ShenandoahFreeSetPartitionId::Mutator) {
+          region->set_update_watermark(region->top<false /*false to avoid loading _atomic_top*/>());
+          region->set_collector_allocator_reserved(false);
+        }
         log_debug(gc, alloc)("%sAllocator: Removing heap region %li from alloc region %i.",
           _alloc_partition_name, region->index(), alloc_region->alloc_region_index);
         alloc_region->address.release_store(nullptr);
@@ -323,10 +320,14 @@ int ShenandoahAllocator<ALLOC_PARTITION>::replenish_alloc_regions(ShenandoahAllo
         if (satisfy_alloc_req_first && reserved[i]->free_words() >= min_req_size) {
           bool ready_for_retire = false;
           *obj = allocate_in<false>(reserved[i], true, *req, *in_new_region, ready_for_retire);
+          reserved[i]->reset_age();
           assert(*obj != nullptr, "Should always succeed");
           satisfy_alloc_req_first = false;
         }
         reserved[i]->set_active_alloc_region();
+        if (ALLOC_PARTITION != ShenandoahFreeSetPartitionId::Mutator) {
+          reserved[i]->set_collector_allocator_reserved(true);
+        }
         log_debug(gc, alloc)("%sAllocator: Storing heap region %li to alloc region %i",
           _alloc_partition_name, reserved[i]->index(), replenishable[i]->alloc_region_index);
         replenishable[i]->address.release_store(reserved[i]);
@@ -371,6 +372,10 @@ void ShenandoahAllocator<ALLOC_PARTITION>::release_alloc_regions(bool should_upd
       log_debug(gc, alloc)("%sAllocator: Releasing heap region %li from alloc region %i",
         _alloc_partition_name, r->index(), i);
       r->unset_active_alloc_region();
+      if (ALLOC_PARTITION != ShenandoahFreeSetPartitionId::Mutator) {
+        r->set_update_watermark(r->top<false /*false to avoid loading _atomic_top*/>());
+        r->set_collector_allocator_reserved(false);
+      }
       alloc_region.address.release_store(nullptr);
       size_t free_bytes = r->free();
       if (free_bytes >= PLAB::min_size_bytes()) {

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp

@@ -470,8 +470,12 @@ void ShenandoahBarrierSet::arraycopy_marking(T* dst, size_t count) {
   }
 }
 
-inline bool ShenandoahBarrierSet::need_bulk_update(HeapWord* ary) {
-  return ary < _heap->heap_region_containing(ary)->get_update_watermark();
+inline bool ShenandoahBarrierSet::need_bulk_update(HeapWord* array) {
+  ShenandoahHeapRegion* r = _heap->heap_region_containing(array);
+  if (r->is_collector_allocator_reserved()) {
+    return true;
+  }
+  return array < r->get_update_watermark();
 }
 
 template <class T>

src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp

@@ -412,6 +412,11 @@ void ShenandoahDegenGC::op_init_update_refs() {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   heap->prepare_update_heap_references();
   heap->set_update_refs_in_progress(true);
+  {
+    // Release alloc regions from allocators for collector.
+    ShenandoahHeapLocker locker(heap->lock());
+    heap->free_set()->collector_allocator()->release_alloc_regions();
+  }
 }
 
 void ShenandoahDegenGC::op_update_refs() {

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -1205,6 +1205,12 @@ void ShenandoahHeap::concurrent_prepare_for_update_refs() {
     set_gc_state_concurrent(UPDATE_REFS, true);
   }
 
+  {
+    // Release alloc regions from allocators for collector.
+    ShenandoahHeapLocker locker(lock());
+    _free_set->collector_allocator()->release_alloc_regions();
+  }
+
   // This will propagate the gc state and retire gclabs and plabs for threads that require it.
   ShenandoahPrepareForUpdateRefsHandshakeClosure prepare_for_update_refs(_gc_state.raw_value());
 

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp

@@ -26,6 +26,7 @@
 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_HPP
 #define SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_HPP
 
+#include "shenandoahAllocator.hpp"
 #include "gc/shared/gc_globals.hpp"
 #include "gc/shared/spaceDecorator.hpp"
 #include "gc/shenandoah/shenandoahAffiliation.hpp"
@@ -267,14 +268,15 @@ private:
 
   Atomic<HeapWord*> _update_watermark;
 
-  volatile uint _age;
+  uint _age;
   bool _promoted_in_place;
-  CENSUS_NOISE(volatile uint _youth;)   // tracks epochs of retrograde ageing (rejuvenation)
+  CENSUS_NOISE(uint _youth;)   // tracks epochs of retrograde ageing (rejuvenation)
 
   ShenandoahSharedFlag _recycling; // Used to indicate that the region is being recycled; see try_recycle*().
 
   bool _needs_bitmap_reset;
 
+  Atomic<bool> _collector_allocator_reserved;
 public:
   ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed);
 
@@ -537,8 +539,8 @@ public:
   void set_affiliation(ShenandoahAffiliation new_affiliation);
 
   // Region ageing and rejuvenation
-  uint age() const { return AtomicAccess::load(&_age); }
-  CENSUS_NOISE(uint youth() const { return AtomicAccess::load(&_youth); })
+  uint age() const { return _age; }
+  CENSUS_NOISE(uint youth() const { return _youth; })
 
   void increment_age() {
     const uint current_age = age();
@@ -550,24 +552,11 @@ public:
   }
 
   void reset_age() {
-    uint current = age();
-    // return immediately in fast path when current age is 0
-    if (current == 0u) return;
-    // reset_age can be called from multiple mutator/worker threads concurrently w/o heap lock,
-    // if no need to update census noise, there is no need to use cmpxchg here.
-    // The while loop with cmpxchg is to make sure we don't duplicately count the age in census noise.
-    uint old = current;
-    while ((current = AtomicAccess::cmpxchg(&_age, old, 0u)) != old &&
-           current != 0u) {
-      old = current;
-    }
-    if (current != 0u) {
-      // Only the thread successfully resets age should update census noise
-      CENSUS_NOISE(AtomicAccess::add(&_youth, current, memory_order_relaxed);)
-    }
+    CENSUS_NOISE(_youth += _age;)
+    _age = 0;
   }
 
-  CENSUS_NOISE(void clear_youth() { AtomicAccess::store(&_youth,  0u); })
+  CENSUS_NOISE(void clear_youth() { _youth = 0u; })
 
   inline bool need_bitmap_reset() const {
     return _needs_bitmap_reset;
@@ -600,6 +589,10 @@ public:
     // meanwhile the previous value of _atomic_top needs to be synced back to _top.
     HeapWord* prior_atomic_top = nullptr;
     HeapWord* current_atomic_top = atomic_top();
+    if (current_atomic_top > _top) {
+      // reset age if there was any allocation in the region after it's reserved as alloc region.
+      reset_age();
+    }
     while (true /*always break out in the loop*/) {
       assert(current_atomic_top != nullptr, "Must not");
       AtomicAccess::store(&_top, current_atomic_top); // Sync current _atomic_top back to _top
@@ -614,11 +607,18 @@ public:
     assert(!is_atomic_alloc_region(), "Must not");
   }
 
-  inline bool is_atomic_alloc_region() const {
+  bool is_atomic_alloc_region() const {
     // region is an active atomic alloc region if the atomic top is set
     return atomic_top() != nullptr;
   }
 
+  void set_collector_allocator_reserved(bool is_collector_allocator_reserved) {
+    _collector_allocator_reserved.release_store(is_collector_allocator_reserved);
+  }
+
+  bool is_collector_allocator_reserved() const {
+    return _collector_allocator_reserved.load_acquire();
+  }
 private:
   void decrement_humongous_waste();
   void do_commit();

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp

@@ -192,7 +192,6 @@ HeapWord* ShenandoahHeapRegion::allocate_lab_atomic(const ShenandoahAllocRequest
     }
     if (adjusted_size >= req.min_size()) {
       if (try_allocate(obj /*value*/, adjusted_size, obj /*reference*/)) {
-        reset_age();
         actual_size = adjusted_size;
         adjust_alloc_metadata(req, adjusted_size);
         ready_for_retire = free_words - adjusted_size < PLAB::min_size();
@@ -312,16 +311,6 @@ inline void ShenandoahHeapRegion::set_update_watermark(HeapWord* w) {
   _update_watermark.release_store(w);
 }
 
-inline void ShenandoahHeapRegion::concurrent_set_update_watermark(HeapWord* w) {
-  assert(bottom() <= w && w <= top(), "within bounds");
-  HeapWord* watermark = nullptr;
-  while ((watermark = _update_watermark.load_acquire()) < w) {
-    if (_update_watermark.compare_exchange( watermark, w, memory_order_release) == watermark) {
-      return;
-    }
-  }
-}
-
 // Fast version that avoids synchronization, only to be used at safepoints.
 inline void ShenandoahHeapRegion::set_update_watermark_at_safepoint(HeapWord* w) {
   assert(bottom() <= w && w <= top(), "within bounds");