From 0fd92d92933626dfbb0fdd5c0d3b804666e2b94e Mon Sep 17 00:00:00 2001 From: Xiaolong Peng Date: Tue, 31 Mar 2026 18:29:07 -0700 Subject: [PATCH] Avoiding calling reset_age and set_update_watermark in fast alloc path --- .../gc/shenandoah/shenandoahAllocator.cpp | 27 +++++++----- .../shenandoahBarrierSet.inline.hpp | 8 +++- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 5 +++ .../share/gc/shenandoah/shenandoahHeap.cpp | 6 +++ .../gc/shenandoah/shenandoahHeapRegion.hpp | 42 +++++++++---------- .../shenandoahHeapRegion.inline.hpp | 11 ----- 6 files changed, 54 insertions(+), 45 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAllocator.cpp b/src/hotspot/share/gc/shenandoah/shenandoahAllocator.cpp index edc0f3b198f..1d2692aeed3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAllocator.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAllocator.cpp @@ -185,6 +185,10 @@ HeapWord* ShenandoahAllocator::attempt_allocation_from_free_set if (r != nullptr) { bool ready_for_retire = false; HeapWord *obj = allocate_in(r, false, req, in_new_region, ready_for_retire); + r->reset_age(); + if (ALLOC_PARTITION != ShenandoahFreeSetPartitionId::Mutator) { + r->set_update_watermark(r->top()); + } assert(obj != nullptr, "Should always succeed."); _free_set->partitions()->increase_used(ALLOC_PARTITION, (req.actual_size() + req.waste()) * HeapWordSize); @@ -258,17 +262,6 @@ HeapWord* ShenandoahAllocator::allocate_in(ShenandoahHeapRegion _alloc_partition_name, actual_size * HeapWordSize, region->index(), req.size() * HeapWordSize, is_alloc_region ? "true" : "false", region->free()); req.set_actual_size(actual_size); in_new_region = obj == region->bottom(); // is in new region when the allocated object is at the bottom of the region. - if (ALLOC_PARTITION != ShenandoahFreeSetPartitionId::Mutator) { - // For GC allocations, we advance update_watermark because the objects relocated into this memory during - // evacuation are not updated during evacuation. For both young and old regions r, it is essential that all - // PLABs be made parsable at the end of evacuation. This is enabled by retiring all plabs at end of evacuation. - if (IS_SHARED_ALLOC_REGION) { - region->concurrent_set_update_watermark(region->top()); - } else { - region->set_update_watermark(region->top()); - } - } - if (!IS_SHARED_ALLOC_REGION && region->free_words() < PLAB::min_size()) { ready_for_retire = true; } @@ -297,6 +290,10 @@ int ShenandoahAllocator::replenish_alloc_regions(ShenandoahAllo if (region == nullptr || free_bytes / HeapWordSize < PLAB::min_size()) { if (region != nullptr) { region->unset_active_alloc_region(); + if (ALLOC_PARTITION != ShenandoahFreeSetPartitionId::Mutator) { + region->set_update_watermark(region->top()); + region->set_collector_allocator_reserved(false); + } log_debug(gc, alloc)("%sAllocator: Removing heap region %li from alloc region %i.", _alloc_partition_name, region->index(), alloc_region->alloc_region_index); alloc_region->address.release_store(nullptr); @@ -323,10 +320,14 @@ int ShenandoahAllocator::replenish_alloc_regions(ShenandoahAllo if (satisfy_alloc_req_first && reserved[i]->free_words() >= min_req_size) { bool ready_for_retire = false; *obj = allocate_in(reserved[i], true, *req, *in_new_region, ready_for_retire); + reserved[i]->reset_age(); assert(*obj != nullptr, "Should always succeed"); satisfy_alloc_req_first = false; } reserved[i]->set_active_alloc_region(); + if (ALLOC_PARTITION != ShenandoahFreeSetPartitionId::Mutator) { + reserved[i]->set_collector_allocator_reserved(true); + } log_debug(gc, alloc)("%sAllocator: Storing heap region %li to alloc region %i", _alloc_partition_name, reserved[i]->index(), replenishable[i]->alloc_region_index); replenishable[i]->address.release_store(reserved[i]); @@ -371,6 +372,10 @@ void ShenandoahAllocator::release_alloc_regions(bool should_upd log_debug(gc, alloc)("%sAllocator: Releasing heap region %li from alloc region %i", _alloc_partition_name, r->index(), i); r->unset_active_alloc_region(); + if (ALLOC_PARTITION != ShenandoahFreeSetPartitionId::Mutator) { + r->set_update_watermark(r->top()); + r->set_collector_allocator_reserved(false); + } alloc_region.address.release_store(nullptr); size_t free_bytes = r->free(); if (free_bytes >= PLAB::min_size_bytes()) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp index f4b35e29b09..a82f3f1b536 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -470,8 +470,12 @@ void ShenandoahBarrierSet::arraycopy_marking(T* dst, size_t count) { } } -inline bool ShenandoahBarrierSet::need_bulk_update(HeapWord* ary) { - return ary < _heap->heap_region_containing(ary)->get_update_watermark(); +inline bool ShenandoahBarrierSet::need_bulk_update(HeapWord* array) { + ShenandoahHeapRegion* r = _heap->heap_region_containing(array); + if (r->is_collector_allocator_reserved()) { + return true; + } + return array < r->get_update_watermark(); } template diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 54aeb712481..4b90aa78fba 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -412,6 +412,11 @@ void ShenandoahDegenGC::op_init_update_refs() { ShenandoahHeap* const heap = ShenandoahHeap::heap(); heap->prepare_update_heap_references(); heap->set_update_refs_in_progress(true); + { + // Release alloc regions from allocators for collector. + ShenandoahHeapLocker locker(heap->lock()); + heap->free_set()->collector_allocator()->release_alloc_regions(); + } } void ShenandoahDegenGC::op_update_refs() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index e38d1574f47..c1b97aa43e3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1205,6 +1205,12 @@ void ShenandoahHeap::concurrent_prepare_for_update_refs() { set_gc_state_concurrent(UPDATE_REFS, true); } + { + // Release alloc regions from allocators for collector. + ShenandoahHeapLocker locker(lock()); + _free_set->collector_allocator()->release_alloc_regions(); + } + // This will propagate the gc state and retire gclabs and plabs for threads that require it. ShenandoahPrepareForUpdateRefsHandshakeClosure prepare_for_update_refs(_gc_state.raw_value()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index cfe02733a6f..ee842fae531 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -26,6 +26,7 @@ #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_HPP #define SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_HPP +#include "shenandoahAllocator.hpp" #include "gc/shared/gc_globals.hpp" #include "gc/shared/spaceDecorator.hpp" #include "gc/shenandoah/shenandoahAffiliation.hpp" @@ -267,14 +268,15 @@ private: Atomic _update_watermark; - volatile uint _age; + uint _age; bool _promoted_in_place; - CENSUS_NOISE(volatile uint _youth;) // tracks epochs of retrograde ageing (rejuvenation) + CENSUS_NOISE(uint _youth;) // tracks epochs of retrograde ageing (rejuvenation) ShenandoahSharedFlag _recycling; // Used to indicate that the region is being recycled; see try_recycle*(). bool _needs_bitmap_reset; + Atomic _collector_allocator_reserved; public: ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed); @@ -537,8 +539,8 @@ public: void set_affiliation(ShenandoahAffiliation new_affiliation); // Region ageing and rejuvenation - uint age() const { return AtomicAccess::load(&_age); } - CENSUS_NOISE(uint youth() const { return AtomicAccess::load(&_youth); }) + uint age() const { return _age; } + CENSUS_NOISE(uint youth() const { return _youth; }) void increment_age() { const uint current_age = age(); @@ -550,24 +552,11 @@ public: } void reset_age() { - uint current = age(); - // return immediately in fast path when current age is 0 - if (current == 0u) return; - // reset_age can be called from multiple mutator/worker threads concurrently w/o heap lock, - // if no need to update census noise, there is no need to use cmpxchg here. - // The while loop with cmpxchg is to make sure we don't duplicately count the age in census noise. - uint old = current; - while ((current = AtomicAccess::cmpxchg(&_age, old, 0u)) != old && - current != 0u) { - old = current; - } - if (current != 0u) { - // Only the thread successfully resets age should update census noise - CENSUS_NOISE(AtomicAccess::add(&_youth, current, memory_order_relaxed);) - } + CENSUS_NOISE(_youth += _age;) + _age = 0; } - CENSUS_NOISE(void clear_youth() { AtomicAccess::store(&_youth, 0u); }) + CENSUS_NOISE(void clear_youth() { _youth = 0u; }) inline bool need_bitmap_reset() const { return _needs_bitmap_reset; @@ -600,6 +589,10 @@ public: // meanwhile the previous value of _atomic_top needs to be synced back to _top. HeapWord* prior_atomic_top = nullptr; HeapWord* current_atomic_top = atomic_top(); + if (current_atomic_top > _top) { + // reset age if there was any allocation in the region after it's reserved as alloc region. + reset_age(); + } while (true /*always break out in the loop*/) { assert(current_atomic_top != nullptr, "Must not"); AtomicAccess::store(&_top, current_atomic_top); // Sync current _atomic_top back to _top @@ -614,11 +607,18 @@ public: assert(!is_atomic_alloc_region(), "Must not"); } - inline bool is_atomic_alloc_region() const { + bool is_atomic_alloc_region() const { // region is an active atomic alloc region if the atomic top is set return atomic_top() != nullptr; } + void set_collector_allocator_reserved(bool is_collector_allocator_reserved) { + _collector_allocator_reserved.release_store(is_collector_allocator_reserved); + } + + bool is_collector_allocator_reserved() const { + return _collector_allocator_reserved.load_acquire(); + } private: void decrement_humongous_waste(); void do_commit(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index 5ac01d3806a..087bc4717da 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -192,7 +192,6 @@ HeapWord* ShenandoahHeapRegion::allocate_lab_atomic(const ShenandoahAllocRequest } if (adjusted_size >= req.min_size()) { if (try_allocate(obj /*value*/, adjusted_size, obj /*reference*/)) { - reset_age(); actual_size = adjusted_size; adjust_alloc_metadata(req, adjusted_size); ready_for_retire = free_words - adjusted_size < PLAB::min_size(); @@ -312,16 +311,6 @@ inline void ShenandoahHeapRegion::set_update_watermark(HeapWord* w) { _update_watermark.release_store(w); } -inline void ShenandoahHeapRegion::concurrent_set_update_watermark(HeapWord* w) { - assert(bottom() <= w && w <= top(), "within bounds"); - HeapWord* watermark = nullptr; - while ((watermark = _update_watermark.load_acquire()) < w) { - if (_update_watermark.compare_exchange( watermark, w, memory_order_release) == watermark) { - return; - } - } -} - // Fast version that avoids synchronization, only to be used at safepoints. inline void ShenandoahHeapRegion::set_update_watermark_at_safepoint(HeapWord* w) { assert(bottom() <= w && w <= top(), "within bounds");