Move thread locals to ShenandoahThreadLocalData

src/hotspot/share/gc/shenandoah/shenandoahAllocator.cpp

@@ -22,7 +22,6 @@
  * questions.
  *
  */
-#include "gc/shared/workerThread.hpp"
 #include "gc/shenandoah/shenandoahAllocator.hpp"
 #include "gc/shenandoah/shenandoahAllocRequest.hpp"
 #include "gc/shenandoah/shenandoahFreeSet.hpp"
@@ -31,7 +30,6 @@
 #include "memory/padded.inline.hpp"
 #include "runtime/atomicAccess.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
-#include "runtime/os.hpp"
 #include "utilities/globalDefinitions.hpp"
 
 template <ShenandoahFreeSetPartitionId ALLOC_PARTITION>
@@ -91,9 +89,45 @@ public:
       }
     }
   }
-
 };
 
+template <ShenandoahFreeSetPartitionId ALLOC_PARTITION>
+uint ShenandoahAllocator<ALLOC_PARTITION>::alloc_start_index() {
+  uint alloc_start_index = 0u;
+  switch (ALLOC_PARTITION) {
+    case ShenandoahFreeSetPartitionId::Mutator:
+      alloc_start_index = ShenandoahThreadLocalData::mutator_allocator_start_index();
+      break;
+    case ShenandoahFreeSetPartitionId::Collector:
+      alloc_start_index = ShenandoahThreadLocalData::collector_allocator_start_index();
+      break;
+    default:
+      break;
+  }
+  if (alloc_start_index == UINT_MAX) {
+    if (_alloc_region_count <= 1u) {
+      alloc_start_index = 0u;
+    } else {
+      if (ALLOC_PARTITION == ShenandoahFreeSetPartitionId::Mutator) {
+        alloc_start_index = abs(os::random()) % _alloc_region_count;
+      } else {
+        alloc_start_index = (Thread::current()->is_Worker_thread() ? WorkerThread::worker_id() : abs(os::random())) % _alloc_region_count;
+      }
+    }
+    switch (ALLOC_PARTITION) {
+      case ShenandoahFreeSetPartitionId::Mutator:
+        ShenandoahThreadLocalData::set_mutator_allocator_start_index(alloc_start_index);
+        break;
+      case ShenandoahFreeSetPartitionId::Collector:
+        ShenandoahThreadLocalData::set_collector_allocator_start_index(alloc_start_index);
+        break;
+      default:
+        break;
+    }
+  }
+  return alloc_start_index;
+}
+
 template <ShenandoahFreeSetPartitionId ALLOC_PARTITION>
 HeapWord* ShenandoahAllocator<ALLOC_PARTITION>::attempt_allocation(ShenandoahAllocRequest& req, bool& in_new_region) {
   // Always allocate under heap lock held when shared alloc region count is set to 0
@@ -384,43 +418,21 @@ void ShenandoahAllocator<ALLOC_PARTITION>::reserve_alloc_regions() {
   }
 }
 
-THREAD_LOCAL uint ShenandoahMutatorAllocator::_alloc_start_index = UINT_MAX;
-
 ShenandoahMutatorAllocator::ShenandoahMutatorAllocator(ShenandoahFreeSet* free_set) :
   ShenandoahAllocator((uint) ShenandoahMutatorAllocRegions, free_set) {
   _yield_to_safepoint = true;
 }
 
-uint ShenandoahMutatorAllocator::alloc_start_index() {
-  if (_alloc_start_index == UINT_MAX) {
-    if (_alloc_region_count <= 1u) {
-      _alloc_start_index = 0u;
-    } else {
-      _alloc_start_index = abs(os::random()) % _alloc_region_count;
-      assert(_alloc_start_index < _alloc_region_count, "alloc_start_index out of range");
-    }
-  }
-  return _alloc_start_index;
-}
-
 ShenandoahCollectorAllocator::ShenandoahCollectorAllocator(ShenandoahFreeSet* free_set) :
   ShenandoahAllocator((uint) ShenandoahCollectorAllocRegions, free_set) {
   _yield_to_safepoint = false;
 }
 
-uint ShenandoahCollectorAllocator::alloc_start_index() {
-  return Thread::current()->is_Worker_thread() ? WorkerThread::worker_id() % _alloc_region_count : 0u;
-}
-
 ShenandoahOldCollectorAllocator::ShenandoahOldCollectorAllocator(ShenandoahFreeSet* free_set) :
   ShenandoahAllocator(0u, free_set) {
   _yield_to_safepoint = false;
 }
 
-uint ShenandoahOldCollectorAllocator::alloc_start_index() {
-  return Thread::current()->is_Worker_thread() ? WorkerThread::worker_id() % _alloc_region_count : 0u;
-}
-
 HeapWord* ShenandoahOldCollectorAllocator::allocate(ShenandoahAllocRequest& req, bool& in_new_region) {
   shenandoah_assert_not_heaplocked();
 #ifdef ASSERT

src/hotspot/share/gc/shenandoah/shenandoahAllocator.hpp

@@ -29,6 +29,7 @@
 #include "gc/shenandoah/shenandoahAllocRequest.hpp"
 #include "gc/shenandoah/shenandoahFreeSetPartitionId.hpp"
 #include "gc/shenandoah/shenandoahPadding.hpp"
+#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
 #include "memory/allocation.hpp"
 #include "memory/padded.hpp"
 #include "runtime/thread.hpp"
@@ -57,7 +58,7 @@ protected:
 
 
   // start index of the shared alloc regions where the allocation will start from.
-  virtual uint alloc_start_index() { return 0u; }
+  uint alloc_start_index();
 
   // Attempt to allocate memory to satisfy alloc request.
   // If _alloc_region_count is not 0, it will try to allocate in shared alloc regions first with atomic operations w/o
@@ -133,14 +134,11 @@ public:
  * Allocator impl for mutator
  */
 class ShenandoahMutatorAllocator : public ShenandoahAllocator<ShenandoahFreeSetPartitionId::Mutator> {
-  static THREAD_LOCAL uint _alloc_start_index;
-  uint alloc_start_index() override;
 public:
   ShenandoahMutatorAllocator(ShenandoahFreeSet* free_set);
 };
 
 class ShenandoahCollectorAllocator : public ShenandoahAllocator<ShenandoahFreeSetPartitionId::Collector> {
-  uint alloc_start_index() override;
 public:
   ShenandoahCollectorAllocator(ShenandoahFreeSet* free_set);
 };
@@ -149,7 +147,6 @@ public:
 // because of the complexity in plab allocation where we have specialized logic to handle card table size alignment.
 // We will make ShenandoahOldCollectorAllocator use compare-and-swap/atomic operation later.
 class ShenandoahOldCollectorAllocator : public ShenandoahAllocator<ShenandoahFreeSetPartitionId::OldCollector> {
-  uint alloc_start_index() override;
 public:
   ShenandoahOldCollectorAllocator(ShenandoahFreeSet* free_set);
   HeapWord* allocate(ShenandoahAllocRequest& req, bool& in_new_region) override;

src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp

@@ -43,7 +43,9 @@ ShenandoahThreadLocalData::ShenandoahThreadLocalData() :
   _plab_promoted(0),
   _plab_allows_promotion(true),
   _plab_retries_enabled(true),
-  _evacuation_stats(new ShenandoahEvacuationStats()) {
+  _evacuation_stats(new ShenandoahEvacuationStats()),
+  _mutator_allocator_start_index(UINT_MAX),
+  _collector_allocator_start_index(UINT_MAX) {
 }
 
 ShenandoahThreadLocalData::~ShenandoahThreadLocalData() {

src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp

@@ -83,6 +83,10 @@ private:
 
   ShenandoahEvacuationStats* _evacuation_stats;
 
+  uint   _mutator_allocator_start_index;
+
+  uint   _collector_allocator_start_index;
+
   ShenandoahThreadLocalData();
   ~ShenandoahThreadLocalData();
 
@@ -280,6 +284,22 @@ public:
   static ByteSize card_table_offset() {
     return Thread::gc_data_offset() + byte_offset_of(ShenandoahThreadLocalData, _card_table);
   }
+
+  static uint mutator_allocator_start_index() {
+    return data(Thread::current())->_mutator_allocator_start_index;
+  }
+
+  static void set_mutator_allocator_start_index(uint start_index) {
+    data(Thread::current())->_mutator_allocator_start_index = start_index;
+  }
+
+  static uint collector_allocator_start_index() {
+    return data(Thread::current())->_collector_allocator_start_index;
+  }
+
+  static void set_collector_allocator_start_index(uint start_index) {
+    data(Thread::current())->_collector_allocator_start_index = start_index;
+  }
 };
 
 STATIC_ASSERT(sizeof(ShenandoahThreadLocalData) <= sizeof(GCThreadLocalData));