From 3435c8dd09945dc6d90274e1318e1e91bc604d51 Mon Sep 17 00:00:00 2001 From: Patrick Fontanilla Date: Mon, 3 Aug 2026 21:27:06 +0000 Subject: [PATCH] Add new fields to ShenandoahThreadLocalData for pin cache and delete ShenandoahRegionPinCache class --- .../gc/shenandoah/shenandoahBarrierSet.cpp | 2 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 47 ++++++++++++-- .../share/gc/shenandoah/shenandoahHeap.hpp | 3 +- .../gc/shenandoah/shenandoahHeapRegion.hpp | 2 +- .../shenandoahHeapRegion.inline.hpp | 2 +- .../shenandoah/shenandoahRegionPinCache.hpp | 53 ---------------- .../shenandoahRegionPinCache.inline.hpp | 63 ------------------- .../shenandoah/shenandoahThreadLocalData.cpp | 4 +- .../shenandoah/shenandoahThreadLocalData.hpp | 19 +++++- 9 files changed, 67 insertions(+), 128 deletions(-) delete mode 100644 src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.hpp delete mode 100644 src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.inline.hpp diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp index 2c213918db7..af41d764066 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp @@ -178,7 +178,7 @@ void ShenandoahBarrierSet::on_thread_detach(Thread *thread) { StackWatermarkSet::finish_processing(JavaThread::cast(thread), &oops, StackWatermarkKind::gc); } - ShenandoahThreadLocalData::pin_count_cache(thread).flush(); + _heap->flush_region_pin_cache(JavaThread::cast(thread)); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 8143f2168b5..0cecc61ba1d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -2436,16 +2436,55 @@ void ShenandoahHeap::unregister_nmethod(nmethod* nm) { } void ShenandoahHeap::pin_object(JavaThread* thr, oop o) { - size_t region_idx = heap_region_index_containing(o); - ShenandoahThreadLocalData::pin_count_cache(thr).inc_count(region_idx); + assert(thr == JavaThread::current(), "Sanity"); + size_t reg_idx_pin = heap_region_index_containing(o); + size_t reg_idx_cached = ShenandoahThreadLocalData::pin_cache_region(thr); + size_t count = ShenandoahThreadLocalData::pin_cache_count(thr); + if (reg_idx_pin == reg_idx_cached) { + ShenandoahThreadLocalData::pin_cache_set_count(thr, count + 1); + } else { + if (count != 0) { + get_region(reg_idx_cached)->inc_pin_count(count); + } + ShenandoahThreadLocalData::pin_cache_set_region(thr, reg_idx_pin); + ShenandoahThreadLocalData::pin_cache_set_count(thr, 1); + } } void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) { - size_t region_idx = heap_region_index_containing(o); - ShenandoahThreadLocalData::pin_count_cache(thr).dec_count(region_idx); + assert(thr == JavaThread::current(), "Sanity"); + size_t reg_idx_pin = heap_region_index_containing(o); + size_t reg_idx_cached = ShenandoahThreadLocalData::pin_cache_region(thr); + size_t count = ShenandoahThreadLocalData::pin_cache_count(thr); + if (reg_idx_pin == reg_idx_cached) { + ShenandoahThreadLocalData::pin_cache_set_count(thr, count - 1); + } else { + if (count != 0) { + get_region(reg_idx_cached)->inc_pin_count(count); + } + ShenandoahThreadLocalData::pin_cache_set_region(thr, reg_idx_pin); + ShenandoahThreadLocalData::pin_cache_set_count(thr, ~(size_t)0); + } +} + +void ShenandoahHeap::flush_region_pin_cache(JavaThread* thr) { + size_t count = ShenandoahThreadLocalData::pin_cache_count(thr); + if (count != 0) { + get_region(ShenandoahThreadLocalData::pin_cache_region(thr))->inc_pin_count(count); + } + ShenandoahThreadLocalData::pin_cache_set_region(thr, SIZE_MAX); + ShenandoahThreadLocalData::pin_cache_set_count(thr, 0); +} + +void ShenandoahHeap::flush_region_pin_cache() { + assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint."); + for (JavaThreadIteratorWithHandle jtiwh; JavaThread *thread = jtiwh.next(); ) { + flush_region_pin_cache(thread); + } } void ShenandoahHeap::sync_pinned_region_status() { + flush_region_pin_cache(); ShenandoahHeapLocker locker(lock()); for (size_t i = 0; i < num_regions(); i++) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index f24b4c2405f..d297f6bf5e3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -686,8 +686,9 @@ public: void pin_object(JavaThread* thread, oop obj) override; void unpin_object(JavaThread* thread, oop obj) override; - // Update all regions pin counts from the per-thread caches and resets them. + // Updates each region's pin counts from the per-thread caches and resets them. // Must be called before any decision based on pin counts. + void flush_region_pin_cache(JavaThread* thread); void flush_region_pin_cache(); void sync_pinned_region_status(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 72021a9fda5..e9fe19313ab 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -227,7 +227,7 @@ public: void record_pin(); void record_unpin(); size_t pin_count() const; - inline void add_pinned_object_count(size_t value); + inline void inc_pin_count(size_t value); private: static size_t RegionCount; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index 34ddf0ea97b..5db88ce013f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -197,7 +197,7 @@ inline void ShenandoahHeapRegion::restore_top_before_promote() { _top_before_promoted = nullptr; } -inline void ShenandoahHeapRegion::add_pinned_object_count(size_t value) { +inline void ShenandoahHeapRegion::inc_pin_count(size_t value) { _critical_pins.add_then_fetch(value, memory_order_relaxed); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.hpp deleted file mode 100644 index d316f4e41e6..00000000000 --- a/src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_HPP -#define SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_HPP - -#include "memory/allocation.hpp" -#include "utilities/globalDefinitions.hpp" - -// Holds (caches) the pending pinned object count adjustment for the region -// _region_idx on a per thread basis. -// Keeping such a cache avoids expensive atomic operations when updating the -// pin count for the very common case that a single thread pins and unpins -// the same object without any interleaving by garbage collection or by -// pinning/unpinning of objects in other regions. -class ShenandoahRegionPinCache : public StackObj { - size_t _region_idx; - size_t _count; - - void flush_and_set(size_t new_region_idx, size_t new_count); - -public: - ShenandoahRegionPinCache() : _region_idx(SIZE_MAX), _count(0) { } - - void inc_count(size_t region_idx); - void dec_count(size_t region_idx); - - void flush(); -}; - -#endif /* SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_HPP */ diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.inline.hpp deleted file mode 100644 index 9ed5f451667..00000000000 --- a/src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.inline.hpp +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_INLINE_HPP -#define SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_INLINE_HPP - -#include "gc/shenandoah/shenandoahRegionPinCache.hpp" - -#include "gc/shenandoah/shenandoahHeap.inline.hpp" - -inline void ShenandoahRegionPinCache::inc_count(size_t region_idx) { - if (region_idx == _region_idx) { - ++_count; - } else { - flush_and_set(region_idx, (size_t)1); - } -} - -inline void ShenandoahRegionPinCache::dec_count(size_t region_idx) { - if (region_idx == _region_idx) { - --_count; - } else { - flush_and_set(region_idx, ~(size_t)0); - } -} - -inline void ShenandoahRegionPinCache::flush_and_set(size_t new_region_idx, size_t new_count) { - if (_count != 0) { - ShenandoahHeapRegion* r = ShenandoahHeap::heap()->get_region(_region_idx); - assert(r != nullptr, "Region %zu must exist", _region_idx); - r->add_pinned_object_count(_count); - } - _region_idx = new_region_idx; - _count = new_count; -} - -inline void ShenandoahRegionPinCache::flush() { - flush_and_set(SIZE_MAX, 0); -} - -#endif /* SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_INLINE_HPP */ diff --git a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp index 6eed66d11b2..eed47667d74 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp @@ -39,7 +39,9 @@ ShenandoahThreadLocalData::ShenandoahThreadLocalData() : _shenandoah_plab(nullptr), _evacuation_stats(new ShenandoahEvacuationStats()), _invisible_root(nullptr), - _invisible_root_word_size(0) { + _invisible_root_word_size(0), + _pin_region_idx(SIZE_MAX), + _pin_count(0) { } ShenandoahThreadLocalData::~ShenandoahThreadLocalData() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp index 118a658c529..44d43d03579 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp @@ -97,7 +97,8 @@ private: // Thread-local pin cache used to increment/decrement the pin count for // a region and flush the accumulated count to the shared pin counter. // This avoids contended atomic updates of the shared pin counter. - ShenandoahRegionPinCache _pin_cache; + size_t _pin_region_idx; + size_t _pin_count; ShenandoahThreadLocalData(); ~ShenandoahThreadLocalData(); @@ -249,8 +250,20 @@ public: return data(thread)->_invisible_root_word_size.load_relaxed(); } - static ShenandoahRegionPinCache& pin_count_cache(Thread* thread) { - return data(thread)->_pin_cache; + static size_t pin_cache_region(Thread* thread) { + return data(thread)->_pin_region_idx; + } + + static size_t pin_cache_count(Thread* thread) { + return data(thread)->_pin_count; + } + + static void pin_cache_set_region(Thread* thread, size_t reg_idx_pin) { + data(thread)->_pin_region_idx = reg_idx_pin; + } + + static void pin_cache_set_count(Thread* thread, size_t new_count) { + data(thread)->_pin_count = new_count; } };