From cd15f5437660fcb933ba3cfc94c40b8417266ff3 Mon Sep 17 00:00:00 2001 From: Patrick Fontanilla Date: Wed, 29 Jul 2026 23:47:38 +0000 Subject: [PATCH] 8388880: Shenandoah: Improve pinning performance --- .../gc/shenandoah/shenandoahBarrierSet.cpp | 3 + .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 1 + .../share/gc/shenandoah/shenandoahFullGC.cpp | 1 + .../gc/shenandoah/shenandoahGeneration.cpp | 1 + .../shenandoah/shenandoahGenerationalHeap.cpp | 1 + .../share/gc/shenandoah/shenandoahHeap.cpp | 17 +++-- .../share/gc/shenandoah/shenandoahHeap.hpp | 4 ++ .../gc/shenandoah/shenandoahHeapRegion.hpp | 1 + .../shenandoahHeapRegion.inline.hpp | 5 +- .../gc/shenandoah/shenandoahOldGeneration.cpp | 1 + .../shenandoah/shenandoahRegionPinCache.hpp | 53 ++++++++++++++++ .../shenandoahRegionPinCache.inline.hpp | 63 +++++++++++++++++++ .../shenandoah/shenandoahThreadLocalData.hpp | 10 +++ 13 files changed, 155 insertions(+), 6 deletions(-) create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.hpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.inline.hpp diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp index 9605eb524cd..2c213918db7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp @@ -30,6 +30,7 @@ #include "gc/shenandoah/shenandoahCardTable.hpp" #include "gc/shenandoah/shenandoahClosures.inline.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahRegionPinCache.inline.hpp" #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" #include "gc/shenandoah/shenandoahStackWatermark.hpp" #ifdef COMPILER1 @@ -176,6 +177,8 @@ void ShenandoahBarrierSet::on_thread_detach(Thread *thread) { ShenandoahContextEvacuateUpdateRootsClosure oops; StackWatermarkSet::finish_processing(JavaThread::cast(thread), &oops, StackWatermarkKind::gc); } + + ShenandoahThreadLocalData::pin_count_cache(thread).flush(); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 3c3cdc4a90a..8b473817393 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -252,6 +252,7 @@ void ShenandoahDegenGC::op_degenerated() { // it, we fail degeneration right away and slide into Full GC to recover. { + heap->flush_region_pin_cache(); heap->sync_pinned_region_status(); heap->collection_set()->clear_current_index(); ShenandoahHeapRegion* r; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index cab0db7e78a..c1d13e9a3a5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -209,6 +209,7 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { rp->abandon_partial_discovery(); // e. Sync pinned region status from the CP marks + heap->flush_region_pin_cache(); heap->sync_pinned_region_status(); if (heap->mode()->is_generational()) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 493736f4194..d243ba2a807 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -258,6 +258,7 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { { ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states : ShenandoahPhaseTimings::degen_gc_final_update_region_states); + heap->flush_region_pin_cache(); ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context()); parallel_heap_region_iterate(&cl); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp index 31129182380..29014e83be4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp @@ -990,6 +990,7 @@ public: }; void ShenandoahGenerationalHeap::final_update_refs_update_region_states() { + flush_region_pin_cache(); ShenandoahSynchronizePinnedRegionStates pins; ShenandoahUpdateRegionAges ages(marking_context()); auto cl = ShenandoahCompositeRegionClosure::of(pins, ages); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 860988a2d7f..8143f2168b5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -71,6 +71,7 @@ #include "gc/shenandoah/shenandoahPartitionAllocator.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" +#include "gc/shenandoah/shenandoahRegionPinCache.inline.hpp" #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" #include "gc/shenandoah/shenandoahSTWMark.hpp" @@ -2435,14 +2436,13 @@ void ShenandoahHeap::unregister_nmethod(nmethod* nm) { } void ShenandoahHeap::pin_object(JavaThread* thr, oop o) { - heap_region_containing(o)->record_pin(); + size_t region_idx = heap_region_index_containing(o); + ShenandoahThreadLocalData::pin_count_cache(thr).inc_count(region_idx); } void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) { - ShenandoahHeapRegion* r = heap_region_containing(o); - assert(r != nullptr, "Sanity"); - assert(r->pin_count() > 0, "Region %zu should have non-zero pins", r->index()); - r->record_unpin(); + size_t region_idx = heap_region_index_containing(o); + ShenandoahThreadLocalData::pin_count_cache(thr).dec_count(region_idx); } void ShenandoahHeap::sync_pinned_region_status() { @@ -2618,6 +2618,7 @@ void ShenandoahHeap::update_heap_region_states(bool concurrent) { } void ShenandoahHeap::final_update_refs_update_region_states() { + flush_region_pin_cache(); ShenandoahSynchronizePinnedRegionStates cl; parallel_heap_region_iterate(&cl); } @@ -2973,3 +2974,9 @@ ShenandoahHeapLocker::ShenandoahHeapLocker(ShenandoahHeapLock* lock, bool allow_ #endif _lock->lock(allow_block_for_safepoint); } + +void ShenandoahHeap::flush_region_pin_cache() { + for (JavaThreadIteratorWithHandle jtiwh; JavaThread *thread = jtiwh.next(); ) { + ShenandoahThreadLocalData::pin_count_cache(thread).flush(); + } +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 171f473d06a..f24b4c2405f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -686,6 +686,10 @@ public: void pin_object(JavaThread* thread, oop obj) override; void unpin_object(JavaThread* thread, oop obj) override; + // Update all regions pin counts from the per-thread caches and resets them. + // Must be called before any decision based on pin counts. + void flush_region_pin_cache(); + void sync_pinned_region_status(); void assert_pinned_region_status() const NOT_DEBUG_RETURN; void assert_pinned_region_status(ShenandoahGeneration* generation) const NOT_DEBUG_RETURN; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index cf60f18e48f..72021a9fda5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -227,6 +227,7 @@ public: void record_pin(); void record_unpin(); size_t pin_count() const; + inline void add_pinned_object_count(size_t value); private: static size_t RegionCount; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index f004fdf0ea2..34ddf0ea97b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -195,7 +195,10 @@ inline void ShenandoahHeapRegion::save_top_before_promote() { inline void ShenandoahHeapRegion::restore_top_before_promote() { _top = _top_before_promoted; _top_before_promoted = nullptr; - } +} +inline void ShenandoahHeapRegion::add_pinned_object_count(size_t value) { + _critical_pins.add_then_fetch(value, memory_order_relaxed); +} #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 0f01795bb4c..e08dba53996 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -453,6 +453,7 @@ void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states : ShenandoahPhaseTimings::degen_gc_final_update_region_states); + heap->flush_region_pin_cache(); ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context()); parallel_heap_region_iterate(&cl); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.hpp new file mode 100644 index 00000000000..8f9b4bba6b0 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.hpp @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_HPP + +#include "memory/allocation.hpp" +#include "utilities/globalDefinitions.hpp" + +// Holds (caches) the pending pinned object count adjustment for the region +// _region_idx on a per thread basis. +// Keeping such a cache avoids the expensive atomic operations when updating the +// pin count for the very common case that the application pins and unpins the +// same object without any interleaving by a garbage collection or pinning/unpinning +// to an object in another region. +class ShenandoahRegionPinCache : public StackObj { + size_t _region_idx; + size_t _count; + + void flush_and_set(size_t new_region_idx, size_t new_count); + +public: + ShenandoahRegionPinCache() : _region_idx(SIZE_MAX), _count(0) { } + + void inc_count(size_t region_idx); + void dec_count(size_t region_idx); + + void flush(); +}; + +#endif /* SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_HPP */ diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.inline.hpp new file mode 100644 index 00000000000..9ed5f451667 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.inline.hpp @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_INLINE_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_INLINE_HPP + +#include "gc/shenandoah/shenandoahRegionPinCache.hpp" + +#include "gc/shenandoah/shenandoahHeap.inline.hpp" + +inline void ShenandoahRegionPinCache::inc_count(size_t region_idx) { + if (region_idx == _region_idx) { + ++_count; + } else { + flush_and_set(region_idx, (size_t)1); + } +} + +inline void ShenandoahRegionPinCache::dec_count(size_t region_idx) { + if (region_idx == _region_idx) { + --_count; + } else { + flush_and_set(region_idx, ~(size_t)0); + } +} + +inline void ShenandoahRegionPinCache::flush_and_set(size_t new_region_idx, size_t new_count) { + if (_count != 0) { + ShenandoahHeapRegion* r = ShenandoahHeap::heap()->get_region(_region_idx); + assert(r != nullptr, "Region %zu must exist", _region_idx); + r->add_pinned_object_count(_count); + } + _region_idx = new_region_idx; + _count = new_count; +} + +inline void ShenandoahRegionPinCache::flush() { + flush_and_set(SIZE_MAX, 0); +} + +#endif /* SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_INLINE_HPP */ diff --git a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp index 1cc11a6f731..118a658c529 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp @@ -37,6 +37,7 @@ #include "gc/shenandoah/shenandoahEvacTracker.hpp" #include "gc/shenandoah/shenandoahGenerationalHeap.hpp" #include "gc/shenandoah/shenandoahPLAB.hpp" +#include "gc/shenandoah/shenandoahRegionPinCache.hpp" #include "gc/shenandoah/shenandoahSATBMarkQueueSet.hpp" #include "runtime/javaThread.hpp" #include "utilities/debug.hpp" @@ -93,6 +94,11 @@ private: Atomic _invisible_root; Atomic _invisible_root_word_size; + // Thread-local pin cache used to increment/decrement the pin count for + // a region and flush the accumulated count to the shared pin counter. + // This avoids contended atomic updates of the shared pin counter. + ShenandoahRegionPinCache _pin_cache; + ShenandoahThreadLocalData(); ~ShenandoahThreadLocalData(); @@ -242,6 +248,10 @@ public: static size_t get_invisible_root_word_size(Thread* thread) { return data(thread)->_invisible_root_word_size.load_relaxed(); } + + static ShenandoahRegionPinCache& pin_count_cache(Thread* thread) { + return data(thread)->_pin_cache; + } }; STATIC_ASSERT(sizeof(ShenandoahThreadLocalData) <= sizeof(GCThreadLocalData));