From c58245644ff7d865eebe98f4b5bdceba96d68818 Mon Sep 17 00:00:00 2001 From: Xiaolong Peng Date: Fri, 9 Jan 2026 22:05:24 -0800 Subject: [PATCH] Move thread locals to ShenandoahThreadLocalData --- .../gc/shenandoah/shenandoahAllocator.cpp | 62 +++++++++++-------- .../gc/shenandoah/shenandoahAllocator.hpp | 7 +-- .../shenandoah/shenandoahThreadLocalData.cpp | 4 +- .../shenandoah/shenandoahThreadLocalData.hpp | 20 ++++++ 4 files changed, 62 insertions(+), 31 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAllocator.cpp b/src/hotspot/share/gc/shenandoah/shenandoahAllocator.cpp index 4c9ff85dbd4..9a42b174267 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAllocator.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAllocator.cpp @@ -22,7 +22,6 @@ * questions. * */ -#include "gc/shared/workerThread.hpp" #include "gc/shenandoah/shenandoahAllocator.hpp" #include "gc/shenandoah/shenandoahAllocRequest.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" @@ -31,7 +30,6 @@ #include "memory/padded.inline.hpp" #include "runtime/atomicAccess.hpp" #include "runtime/interfaceSupport.inline.hpp" -#include "runtime/os.hpp" #include "utilities/globalDefinitions.hpp" template @@ -91,9 +89,45 @@ public: } } } - }; +template +uint ShenandoahAllocator::alloc_start_index() { + uint alloc_start_index = 0u; + switch (ALLOC_PARTITION) { + case ShenandoahFreeSetPartitionId::Mutator: + alloc_start_index = ShenandoahThreadLocalData::mutator_allocator_start_index(); + break; + case ShenandoahFreeSetPartitionId::Collector: + alloc_start_index = ShenandoahThreadLocalData::collector_allocator_start_index(); + break; + default: + break; + } + if (alloc_start_index == UINT_MAX) { + if (_alloc_region_count <= 1u) { + alloc_start_index = 0u; + } else { + if (ALLOC_PARTITION == ShenandoahFreeSetPartitionId::Mutator) { + alloc_start_index = abs(os::random()) % _alloc_region_count; + } else { + alloc_start_index = (Thread::current()->is_Worker_thread() ? WorkerThread::worker_id() : abs(os::random())) % _alloc_region_count; + } + } + switch (ALLOC_PARTITION) { + case ShenandoahFreeSetPartitionId::Mutator: + ShenandoahThreadLocalData::set_mutator_allocator_start_index(alloc_start_index); + break; + case ShenandoahFreeSetPartitionId::Collector: + ShenandoahThreadLocalData::set_collector_allocator_start_index(alloc_start_index); + break; + default: + break; + } + } + return alloc_start_index; +} + template HeapWord* ShenandoahAllocator::attempt_allocation(ShenandoahAllocRequest& req, bool& in_new_region) { // Always allocate under heap lock held when shared alloc region count is set to 0 @@ -384,43 +418,21 @@ void ShenandoahAllocator::reserve_alloc_regions() { } } -THREAD_LOCAL uint ShenandoahMutatorAllocator::_alloc_start_index = UINT_MAX; - ShenandoahMutatorAllocator::ShenandoahMutatorAllocator(ShenandoahFreeSet* free_set) : ShenandoahAllocator((uint) ShenandoahMutatorAllocRegions, free_set) { _yield_to_safepoint = true; } -uint ShenandoahMutatorAllocator::alloc_start_index() { - if (_alloc_start_index == UINT_MAX) { - if (_alloc_region_count <= 1u) { - _alloc_start_index = 0u; - } else { - _alloc_start_index = abs(os::random()) % _alloc_region_count; - assert(_alloc_start_index < _alloc_region_count, "alloc_start_index out of range"); - } - } - return _alloc_start_index; -} - ShenandoahCollectorAllocator::ShenandoahCollectorAllocator(ShenandoahFreeSet* free_set) : ShenandoahAllocator((uint) ShenandoahCollectorAllocRegions, free_set) { _yield_to_safepoint = false; } -uint ShenandoahCollectorAllocator::alloc_start_index() { - return Thread::current()->is_Worker_thread() ? WorkerThread::worker_id() % _alloc_region_count : 0u; -} - ShenandoahOldCollectorAllocator::ShenandoahOldCollectorAllocator(ShenandoahFreeSet* free_set) : ShenandoahAllocator(0u, free_set) { _yield_to_safepoint = false; } -uint ShenandoahOldCollectorAllocator::alloc_start_index() { - return Thread::current()->is_Worker_thread() ? WorkerThread::worker_id() % _alloc_region_count : 0u; -} - HeapWord* ShenandoahOldCollectorAllocator::allocate(ShenandoahAllocRequest& req, bool& in_new_region) { shenandoah_assert_not_heaplocked(); #ifdef ASSERT diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAllocator.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAllocator.hpp index d10375173f0..f8a9f260d94 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAllocator.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAllocator.hpp @@ -29,6 +29,7 @@ #include "gc/shenandoah/shenandoahAllocRequest.hpp" #include "gc/shenandoah/shenandoahFreeSetPartitionId.hpp" #include "gc/shenandoah/shenandoahPadding.hpp" +#include "gc/shenandoah/shenandoahThreadLocalData.hpp" #include "memory/allocation.hpp" #include "memory/padded.hpp" #include "runtime/thread.hpp" @@ -57,7 +58,7 @@ protected: // start index of the shared alloc regions where the allocation will start from. - virtual uint alloc_start_index() { return 0u; } + uint alloc_start_index(); // Attempt to allocate memory to satisfy alloc request. // If _alloc_region_count is not 0, it will try to allocate in shared alloc regions first with atomic operations w/o @@ -133,14 +134,11 @@ public: * Allocator impl for mutator */ class ShenandoahMutatorAllocator : public ShenandoahAllocator { - static THREAD_LOCAL uint _alloc_start_index; - uint alloc_start_index() override; public: ShenandoahMutatorAllocator(ShenandoahFreeSet* free_set); }; class ShenandoahCollectorAllocator : public ShenandoahAllocator { - uint alloc_start_index() override; public: ShenandoahCollectorAllocator(ShenandoahFreeSet* free_set); }; @@ -149,7 +147,6 @@ public: // because of the complexity in plab allocation where we have specialized logic to handle card table size alignment. // We will make ShenandoahOldCollectorAllocator use compare-and-swap/atomic operation later. class ShenandoahOldCollectorAllocator : public ShenandoahAllocator { - uint alloc_start_index() override; public: ShenandoahOldCollectorAllocator(ShenandoahFreeSet* free_set); HeapWord* allocate(ShenandoahAllocRequest& req, bool& in_new_region) override; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp index ace5ab5e69a..810015f292c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp @@ -43,7 +43,9 @@ ShenandoahThreadLocalData::ShenandoahThreadLocalData() : _plab_promoted(0), _plab_allows_promotion(true), _plab_retries_enabled(true), - _evacuation_stats(new ShenandoahEvacuationStats()) { + _evacuation_stats(new ShenandoahEvacuationStats()), + _mutator_allocator_start_index(UINT_MAX), + _collector_allocator_start_index(UINT_MAX) { } ShenandoahThreadLocalData::~ShenandoahThreadLocalData() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp index f54a65b0785..48282c326ab 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp @@ -83,6 +83,10 @@ private: ShenandoahEvacuationStats* _evacuation_stats; + uint _mutator_allocator_start_index; + + uint _collector_allocator_start_index; + ShenandoahThreadLocalData(); ~ShenandoahThreadLocalData(); @@ -280,6 +284,22 @@ public: static ByteSize card_table_offset() { return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _card_table); } + + static uint mutator_allocator_start_index() { + return data(Thread::current())->_mutator_allocator_start_index; + } + + static void set_mutator_allocator_start_index(uint start_index) { + data(Thread::current())->_mutator_allocator_start_index = start_index; + } + + static uint collector_allocator_start_index() { + return data(Thread::current())->_collector_allocator_start_index; + } + + static void set_collector_allocator_start_index(uint start_index) { + data(Thread::current())->_collector_allocator_start_index = start_index; + } }; STATIC_ASSERT(sizeof(ShenandoahThreadLocalData) <= sizeof(GCThreadLocalData));