8388880: Shenandoah: Improve pinning performance

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp

@@ -30,6 +30,7 @@
 #include "gc/shenandoah/shenandoahCardTable.hpp"
 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahRegionPinCache.inline.hpp"
 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
 #ifdef COMPILER1
@@ -176,6 +177,8 @@ void ShenandoahBarrierSet::on_thread_detach(Thread *thread) {
       ShenandoahContextEvacuateUpdateRootsClosure oops;
       StackWatermarkSet::finish_processing(JavaThread::cast(thread), &oops, StackWatermarkKind::gc);
     }
+
+    ShenandoahThreadLocalData::pin_count_cache(thread).flush();
   }
 }
 

src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp

@@ -252,6 +252,7 @@ void ShenandoahDegenGC::op_degenerated() {
         // it, we fail degeneration right away and slide into Full GC to recover.
 
         {
+          heap->flush_region_pin_cache();
           heap->sync_pinned_region_status();
           heap->collection_set()->clear_current_index();
           ShenandoahHeapRegion* r;

src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp

@@ -209,6 +209,7 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {
     rp->abandon_partial_discovery();
 
     // e. Sync pinned region status from the CP marks
+    heap->flush_region_pin_cache();
     heap->sync_pinned_region_status();
 
     if (heap->mode()->is_generational()) {

src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp

@@ -258,6 +258,7 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) {
   {
     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states :
                             ShenandoahPhaseTimings::degen_gc_final_update_region_states);
+    heap->flush_region_pin_cache();
     ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context());
     parallel_heap_region_iterate(&cl);
 

src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp

@@ -990,6 +990,7 @@ public:
 };
 
 void ShenandoahGenerationalHeap::final_update_refs_update_region_states() {
+  flush_region_pin_cache();
   ShenandoahSynchronizePinnedRegionStates pins;
   ShenandoahUpdateRegionAges ages(marking_context());
   auto cl = ShenandoahCompositeRegionClosure::of(pins, ages);

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -71,6 +71,7 @@
 #include "gc/shenandoah/shenandoahPartitionAllocator.hpp"
 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
+#include "gc/shenandoah/shenandoahRegionPinCache.inline.hpp"
 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
 #include "gc/shenandoah/shenandoahSTWMark.hpp"
@@ -2435,14 +2436,13 @@ void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
 }
 
 void ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
-  heap_region_containing(o)->record_pin();
+  size_t region_idx = heap_region_index_containing(o);
+  ShenandoahThreadLocalData::pin_count_cache(thr).inc_count(region_idx);
 }
 
 void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
-  ShenandoahHeapRegion* r = heap_region_containing(o);
-  assert(r != nullptr, "Sanity");
-  assert(r->pin_count() > 0, "Region %zu should have non-zero pins", r->index());
-  r->record_unpin();
+  size_t region_idx = heap_region_index_containing(o);
+  ShenandoahThreadLocalData::pin_count_cache(thr).dec_count(region_idx);
 }
 
 void ShenandoahHeap::sync_pinned_region_status() {
@@ -2618,6 +2618,7 @@ void ShenandoahHeap::update_heap_region_states(bool concurrent) {
 }
 
 void ShenandoahHeap::final_update_refs_update_region_states() {
+  flush_region_pin_cache();
   ShenandoahSynchronizePinnedRegionStates cl;
   parallel_heap_region_iterate(&cl);
 }
@@ -2973,3 +2974,9 @@ ShenandoahHeapLocker::ShenandoahHeapLocker(ShenandoahHeapLock* lock, bool allow_
 #endif
   _lock->lock(allow_block_for_safepoint);
 }
+
+void ShenandoahHeap::flush_region_pin_cache() {
+  for (JavaThreadIteratorWithHandle jtiwh; JavaThread *thread = jtiwh.next(); ) {
+    ShenandoahThreadLocalData::pin_count_cache(thread).flush();
+  }
+}

src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp

@@ -686,6 +686,10 @@ public:
   void pin_object(JavaThread* thread, oop obj) override;
   void unpin_object(JavaThread* thread, oop obj) override;
 
+  // Update all regions pin counts from the per-thread caches and resets them.
+  // Must be called before any decision based on pin counts.
+  void flush_region_pin_cache();
+
   void sync_pinned_region_status();
   void assert_pinned_region_status() const NOT_DEBUG_RETURN;
   void assert_pinned_region_status(ShenandoahGeneration* generation) const NOT_DEBUG_RETURN;

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp

@@ -227,6 +227,7 @@ public:
   void record_pin();
   void record_unpin();
   size_t pin_count() const;
+  inline void add_pinned_object_count(size_t value);
 
 private:
   static size_t RegionCount;

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp

@@ -195,7 +195,10 @@ inline void ShenandoahHeapRegion::save_top_before_promote() {
 inline void ShenandoahHeapRegion::restore_top_before_promote() {
   _top = _top_before_promoted;
   _top_before_promoted = nullptr;
- }
+}
 
+inline void ShenandoahHeapRegion::add_pinned_object_count(size_t value) {
+  _critical_pins.add_then_fetch(value, memory_order_relaxed);
+}
 
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_INLINE_HPP

src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp

@@ -453,6 +453,7 @@ void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent
     ShenandoahGCPhase phase(concurrent ?
         ShenandoahPhaseTimings::final_update_region_states :
         ShenandoahPhaseTimings::degen_gc_final_update_region_states);
+    heap->flush_region_pin_cache();
     ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context());
 
     parallel_heap_region_iterate(&cl);

src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.hpp

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_HPP
+
+#include "memory/allocation.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+// Holds (caches) the pending pinned object count adjustment for the region
+// _region_idx on a per thread basis.
+// Keeping such a cache avoids the expensive atomic operations when updating the
+// pin count for the very common case that the application pins and unpins the
+// same object without any interleaving by a garbage collection or pinning/unpinning
+// to an object in another region.
+class ShenandoahRegionPinCache : public StackObj {
+  size_t _region_idx;
+  size_t _count;
+
+  void flush_and_set(size_t new_region_idx, size_t new_count);
+
+public:
+  ShenandoahRegionPinCache() : _region_idx(SIZE_MAX), _count(0) { }
+
+  void inc_count(size_t region_idx);
+  void dec_count(size_t region_idx);
+
+  void flush();
+};
+
+#endif /* SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_HPP */

src/hotspot/share/gc/shenandoah/shenandoahRegionPinCache.inline.hpp

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_INLINE_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_INLINE_HPP
+
+#include "gc/shenandoah/shenandoahRegionPinCache.hpp"
+
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+
+inline void ShenandoahRegionPinCache::inc_count(size_t region_idx) {
+  if (region_idx == _region_idx) {
+    ++_count;
+  } else {
+    flush_and_set(region_idx, (size_t)1);
+  }
+}
+
+inline void ShenandoahRegionPinCache::dec_count(size_t region_idx) {
+  if (region_idx == _region_idx) {
+    --_count;
+  } else {
+    flush_and_set(region_idx, ~(size_t)0);
+  }
+}
+
+inline void ShenandoahRegionPinCache::flush_and_set(size_t new_region_idx, size_t new_count) {
+  if (_count != 0) {
+    ShenandoahHeapRegion* r = ShenandoahHeap::heap()->get_region(_region_idx);
+    assert(r != nullptr, "Region %zu must exist", _region_idx);
+    r->add_pinned_object_count(_count);
+  }
+  _region_idx = new_region_idx;
+  _count = new_count;
+}
+
+inline void ShenandoahRegionPinCache::flush() {
+  flush_and_set(SIZE_MAX, 0);
+}
+
+#endif /* SHARE_GC_SHENANDOAH_SHENANDOAHREGIONPINCACHE_INLINE_HPP */

src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp

@@ -37,6 +37,7 @@
 #include "gc/shenandoah/shenandoahEvacTracker.hpp"
 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
 #include "gc/shenandoah/shenandoahPLAB.hpp"
+#include "gc/shenandoah/shenandoahRegionPinCache.hpp"
 #include "gc/shenandoah/shenandoahSATBMarkQueueSet.hpp"
 #include "runtime/javaThread.hpp"
 #include "utilities/debug.hpp"
@@ -93,6 +94,11 @@ private:
   Atomic<HeapWord*> _invisible_root;
   Atomic<size_t> _invisible_root_word_size;
 
+  // Thread-local pin cache used to increment/decrement the pin count for
+  // a region and flush the accumulated count to the shared pin counter.
+  // This avoids contended atomic updates of the shared pin counter.
+  ShenandoahRegionPinCache _pin_cache;
+
   ShenandoahThreadLocalData();
   ~ShenandoahThreadLocalData();
 
@@ -242,6 +248,10 @@ public:
   static size_t get_invisible_root_word_size(Thread* thread) {
     return data(thread)->_invisible_root_word_size.load_relaxed();
   }
+
+  static ShenandoahRegionPinCache& pin_count_cache(Thread* thread) {
+    return data(thread)->_pin_cache;
+  }
 };
 
 STATIC_ASSERT(sizeof(ShenandoahThreadLocalData) <= sizeof(GCThreadLocalData));