8385592: Shenandoah: Encapsulate memory allocation path into ShenandoahAllocator interface

src/hotspot/share/gc/shenandoah/shenandoahAllocator.hpp

@@ -0,0 +1,49 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE THIS COPYRIGHT NOTICE OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHALLOCATOR_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHALLOCATOR_HPP
+
+#include "memory/allocation.hpp"
+
+class ShenandoahAllocRequest;
+class ShenandoahFreeSet;
+
+// ShenandoahAllocator defines the allocation interface for the Shenandoah GC.
+// Subclasses implement different allocation strategies (e.g. serial under heap lock, CAS-based).
+// Partition accounting is delegated back to ShenandoahFreeSet.
+class ShenandoahAllocator : public CHeapObj<mtGC> {
+protected:
+  ShenandoahFreeSet* const _free_set;
+
+public:
+  ShenandoahAllocator(ShenandoahFreeSet* free_set) : _free_set(free_set) {}
+  virtual ~ShenandoahAllocator() = default;
+
+  // Allocate memory for the given request. Returns nullptr on failure.
+  // Sets in_new_region to true if allocation consumes a previously empty region.
+  virtual HeapWord* allocate(ShenandoahAllocRequest& req, bool& in_new_region) = 0;
+};
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHALLOCATOR_HPP

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp

@@ -222,18 +222,18 @@ ShenandoahFreeSetPartitionId ShenandoahFreeSet::prepare_to_promote_in_place(size
   return p;
 }
 
-inline bool ShenandoahFreeSet::can_allocate_from(ShenandoahHeapRegion *r) const {
+bool ShenandoahFreeSet::can_allocate_from(ShenandoahHeapRegion *r) const {
   const auto state = r->state();
   return ShenandoahHeapRegion::is_empty_state(state)
       || (ShenandoahHeapRegion::is_trash(state) && !_heap->is_concurrent_weak_root_in_progress());
 }
 
-inline bool ShenandoahFreeSet::can_allocate_from(size_t idx) const {
+bool ShenandoahFreeSet::can_allocate_from(size_t idx) const {
   ShenandoahHeapRegion* r = _heap->get_region(idx);
   return can_allocate_from(r);
 }
 
-inline size_t ShenandoahFreeSet::alloc_capacity(ShenandoahHeapRegion *r) const {
+size_t ShenandoahFreeSet::alloc_capacity(ShenandoahHeapRegion *r) const {
   if (r->is_trash()) {
     // This would be recycled on allocation path
     return ShenandoahHeapRegion::region_size_bytes();
@@ -242,12 +242,12 @@ inline size_t ShenandoahFreeSet::alloc_capacity(ShenandoahHeapRegion *r) const {
   }
 }
 
-inline size_t ShenandoahFreeSet::alloc_capacity(size_t idx) const {
+size_t ShenandoahFreeSet::alloc_capacity(size_t idx) const {
   ShenandoahHeapRegion* r = _heap->get_region(idx);
   return alloc_capacity(r);
 }
 
-inline bool ShenandoahFreeSet::has_alloc_capacity(ShenandoahHeapRegion *r) const {
+bool ShenandoahFreeSet::has_alloc_capacity(ShenandoahHeapRegion *r) const {
   return alloc_capacity(r) > 0;
 }
 
@@ -315,7 +315,48 @@ void ShenandoahFreeSet::increase_bytes_allocated(size_t bytes) {
   _mutator_bytes_allocated_since_gc_start += bytes;
 }
 
-inline idx_t ShenandoahRegionPartitions::leftmost(ShenandoahFreeSetPartitionId which_partition) const {
+void ShenandoahFreeSet::notify_allocation(ShenandoahFreeSetPartitionId partition, bool in_new_region) {
+  switch (partition) {
+  case ShenandoahFreeSetPartitionId::Mutator:
+    recompute_total_used</* UsedByMutatorChanged */ true,
+                         /* UsedByCollectorChanged */ false, /* UsedByOldCollectorChanged */ false>();
+    if (in_new_region) {
+      recompute_total_affiliated</* MutatorEmptiesChanged */ true, /* CollectorEmptiesChanged */ false,
+                                 /* OldCollectorEmptiesChanged */ false, /* MutatorSizeChanged */ false,
+                                 /* CollectorSizeChanged */ false, /* OldCollectorSizeChanged */ false,
+                                 /* AffiliatedChangesAreYoungNeutral */ false, /* AffiliatedChangesAreGlobalNeutral */ false,
+                                 /* UnaffiliatedChangesAreYoungNeutral */ false>();
+    }
+    break;
+  case ShenandoahFreeSetPartitionId::Collector:
+    recompute_total_used</* UsedByMutatorChanged */ false,
+                         /* UsedByCollectorChanged */ true, /* UsedByOldCollectorChanged */ false>();
+    if (in_new_region) {
+      recompute_total_affiliated</* MutatorEmptiesChanged */ false, /* CollectorEmptiesChanged */ true,
+                                 /* OldCollectorEmptiesChanged */ false, /* MutatorSizeChanged */ false,
+                                 /* CollectorSizeChanged */ false, /* OldCollectorSizeChanged */ false,
+                                 /* AffiliatedChangesAreYoungNeutral */ false, /* AffiliatedChangesAreGlobalNeutral */ false,
+                                 /* UnaffiliatedChangesAreYoungNeutral */ false>();
+    }
+    break;
+  case ShenandoahFreeSetPartitionId::OldCollector:
+    recompute_total_used</* UsedByMutatorChanged */ false,
+                         /* UsedByCollectorChanged */ false, /* UsedByOldCollectorChanged */ true>();
+    if (in_new_region) {
+      recompute_total_affiliated</* MutatorEmptiesChanged */ false, /* CollectorEmptiesChanged */ false,
+                                 /* OldCollectorEmptiesChanged */ true, /* MutatorSizeChanged */ false,
+                                 /* CollectorSizeChanged */ false, /* OldCollectorSizeChanged */ false,
+                                 /* AffiliatedChangesAreYoungNeutral */ true, /* AffiliatedChangesAreGlobalNeutral */ false,
+                                 /* UnaffiliatedChangesAreYoungNeutral */ true>();
+    }
+    break;
+  case ShenandoahFreeSetPartitionId::NotFree:
+  default:
+    assert(false, "won't happen");
+  }
+}
+
+idx_t ShenandoahRegionPartitions::leftmost(ShenandoahFreeSetPartitionId which_partition) const {
   assert (which_partition < NumPartitions, "selected free partition must be valid");
   idx_t idx = _leftmosts[int(which_partition)];
   if (idx >= _max) {
@@ -328,7 +369,7 @@ inline idx_t ShenandoahRegionPartitions::leftmost(ShenandoahFreeSetPartitionId w
   }
 }
 
-inline idx_t ShenandoahRegionPartitions::rightmost(ShenandoahFreeSetPartitionId which_partition) const {
+idx_t ShenandoahRegionPartitions::rightmost(ShenandoahFreeSetPartitionId which_partition) const {
   assert (which_partition < NumPartitions, "selected free partition must be valid");
   idx_t idx = _rightmosts[int(which_partition)];
   // Cannot assert that membership[which_partition.is_set(idx) because this helper method may be used
@@ -819,12 +860,12 @@ inline bool ShenandoahRegionPartitions::partition_id_matches(idx_t idx, Shenando
 }
 #endif
 
-inline bool ShenandoahRegionPartitions::is_empty(ShenandoahFreeSetPartitionId which_partition) const {
+bool ShenandoahRegionPartitions::is_empty(ShenandoahFreeSetPartitionId which_partition) const {
   assert (which_partition < NumPartitions, "selected free partition must be valid");
   return (leftmost(which_partition) > rightmost(which_partition));
 }
 
-inline idx_t ShenandoahRegionPartitions::find_index_of_next_available_region(
+idx_t ShenandoahRegionPartitions::find_index_of_next_available_region(
   ShenandoahFreeSetPartitionId which_partition, idx_t start_index) const {
   idx_t rightmost_idx = rightmost(which_partition);
   idx_t leftmost_idx = leftmost(which_partition);
@@ -840,7 +881,7 @@ inline idx_t ShenandoahRegionPartitions::find_index_of_next_available_region(
   return result;
 }
 
-inline idx_t ShenandoahRegionPartitions::find_index_of_previous_available_region(
+idx_t ShenandoahRegionPartitions::find_index_of_previous_available_region(
   ShenandoahFreeSetPartitionId which_partition, idx_t last_index) const {
   idx_t rightmost_idx = rightmost(which_partition);
   idx_t leftmost_idx = leftmost(which_partition);
@@ -1213,7 +1254,7 @@ void ShenandoahRegionPartitions::assert_bounds() {
          "Mutator humongous waste must match");
 }
 
-inline void ShenandoahRegionPartitions::assert_bounds_sanity() {
+void ShenandoahRegionPartitions::assert_bounds_sanity() {
   for (uint8_t i = 0; i < UIntNumPartitions; i++) {
     ShenandoahFreeSetPartitionId partition = static_cast<ShenandoahFreeSetPartitionId>(i);
     assert(leftmost(partition) == _max || membership(leftmost(partition)) == partition, "Left most boundry must be sane");

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp

@@ -35,6 +35,9 @@
 typedef ShenandoahLock                           ShenandoahRebuildLock;
 typedef ShenandoahLocker<ShenandoahRebuildLock>  ShenandoahRebuildLocker;
 
+class ShenandoahAllocator;
+class ShenandoahSerialAllocator;
+
 // Each ShenandoahHeapRegion is associated with a ShenandoahFreeSetPartitionId.
 enum class ShenandoahFreeSetPartitionId : uint8_t {
   Mutator,                      // Region is in the Mutator free set: available memory is available to mutators.
@@ -161,7 +164,7 @@ public:
     _membership[int(p)].clear_bit(idx);
   }
 
-  inline void one_region_is_no_longer_empty(ShenandoahFreeSetPartitionId partition);
+  void one_region_is_no_longer_empty(ShenandoahFreeSetPartitionId partition);
 
   // Set the Mutator intervals, usage, and capacity according to arguments.  Reset the Collector intervals, used, capacity
   // to represent empty Collector free set.  We use this at the end of rebuild_free_set() to avoid the overhead of making
@@ -231,11 +234,11 @@ public:
   const char* partition_membership_name(idx_t idx) const;
 
   // Return the index of the next available region >= start_index, or maximum_regions if not found.
-  inline idx_t find_index_of_next_available_region(ShenandoahFreeSetPartitionId which_partition,
+  idx_t find_index_of_next_available_region(ShenandoahFreeSetPartitionId which_partition,
                                                         idx_t start_index) const;
 
   // Return the index of the previous available region <= last_index, or -1 if not found.
-  inline idx_t find_index_of_previous_available_region(ShenandoahFreeSetPartitionId which_partition,
+  idx_t find_index_of_previous_available_region(ShenandoahFreeSetPartitionId which_partition,
                                                             idx_t last_index) const;
 
   // Return the index of the next available cluster of cluster_size regions >= start_index, or maximum_regions if not found.
@@ -279,12 +282,12 @@ public:
   //     leftmost() and leftmost_empty() return _max, rightmost() and rightmost_empty() return 0
   //   otherwise, expect the following:
   //     0 <= leftmost <= leftmost_empty <= rightmost_empty <= rightmost < _max
-  inline idx_t leftmost(ShenandoahFreeSetPartitionId which_partition) const;
-  inline idx_t rightmost(ShenandoahFreeSetPartitionId which_partition) const;
+  idx_t leftmost(ShenandoahFreeSetPartitionId which_partition) const;
+  idx_t rightmost(ShenandoahFreeSetPartitionId which_partition) const;
   idx_t leftmost_empty(ShenandoahFreeSetPartitionId which_partition);
   idx_t rightmost_empty(ShenandoahFreeSetPartitionId which_partition);
 
-  inline bool is_empty(ShenandoahFreeSetPartitionId which_partition) const;
+  bool is_empty(ShenandoahFreeSetPartitionId which_partition) const;
 
   inline void increase_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions);
   inline void decrease_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions);
@@ -311,7 +314,7 @@ public:
   inline void decrease_available(ShenandoahFreeSetPartitionId which_partition, size_t bytes);
   inline size_t get_available(ShenandoahFreeSetPartitionId which_partition);
 
-  inline void increase_used(ShenandoahFreeSetPartitionId which_partition, size_t bytes);
+  void increase_used(ShenandoahFreeSetPartitionId which_partition, size_t bytes);
   inline void decrease_used(ShenandoahFreeSetPartitionId which_partition, size_t bytes);
   inline size_t get_used(ShenandoahFreeSetPartitionId which_partition) {
     assert (which_partition < NumPartitions, "Partition must be valid");
@@ -404,7 +407,7 @@ public:
   void assert_bounds() NOT_DEBUG_RETURN;
   // this checks certain sanity conditions related to the bounds with much less effort than is required to
   // more rigorously enforce correctness as is done by assert_bounds()
-  inline void assert_bounds_sanity() NOT_DEBUG_RETURN;
+  void assert_bounds_sanity() NOT_DEBUG_RETURN;
 };
 
 // Publicly, ShenandoahFreeSet represents memory that is available to mutator threads.  The public capacity(), used(),
@@ -432,6 +435,9 @@ public:
 //     during the next GC pass.
 
 class ShenandoahFreeSet : public CHeapObj<mtGC> {
+  friend class ShenandoahAllocator;
+  friend class ShenandoahSerialAllocator;
+
 using idx_t = ShenandoahSimpleBitMap::idx_t;
 private:
   ShenandoahHeap* const _heap;
@@ -622,10 +628,10 @@ private:
   // Returns true iff this region is entirely available, either because it is empty() or because it has been found to represent
   // immediate trash and we'll be able to immediately recycle it.  Note that we cannot recycle immediate trash if
   // concurrent weak root processing is in progress.
-  inline bool can_allocate_from(ShenandoahHeapRegion *r) const;
-  inline bool can_allocate_from(size_t idx) const;
+  bool can_allocate_from(ShenandoahHeapRegion *r) const;
+  bool can_allocate_from(size_t idx) const;
 
-  inline bool has_alloc_capacity(ShenandoahHeapRegion *r) const;
+  bool has_alloc_capacity(ShenandoahHeapRegion *r) const;
 
   void transfer_empty_regions_from_to(ShenandoahFreeSetPartitionId source_partition,
                                       ShenandoahFreeSetPartitionId dest_partition,
@@ -668,6 +674,9 @@ public:
 
   void increase_bytes_allocated(size_t bytes);
 
+  // Called by ShenandoahAllocator after a successful allocation to update used/affiliated totals.
+  void notify_allocation(ShenandoahFreeSetPartitionId partition, bool in_new_region);
+
   // Return an approximation of the bytes allocated since GC start.  The value returned is monotonically non-decreasing
   // in time within each GC cycle.  For certain GC cycles, the value returned may include some bytes allocated before
   // the start of the current GC cycle.
@@ -696,8 +705,8 @@ public:
   }
 
   // Public because ShenandoahRegionPartitions assertions require access.
-  inline size_t alloc_capacity(ShenandoahHeapRegion *r) const;
-  inline size_t alloc_capacity(size_t idx) const;
+  size_t alloc_capacity(ShenandoahHeapRegion *r) const;
+  size_t alloc_capacity(size_t idx) const;
 
   // Return bytes used by old
   inline size_t old_used() {

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -43,6 +43,7 @@
 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
 #include "gc/shenandoah/shenandoahAllocRequest.hpp"
+#include "gc/shenandoah/shenandoahSerialAllocator.hpp"
 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
@@ -436,6 +437,7 @@ jint ShenandoahHeap::initialize() {
     }
 
     _free_set = new ShenandoahFreeSet(this, _num_regions);
+    _allocator = new ShenandoahSerialAllocator(_free_set);
     initialize_generations();
 
     // We are initializing free set.  We ignore cset region tallies.
@@ -576,6 +578,7 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
   _shenandoah_policy(policy),
   _gc_mode(nullptr),
   _free_set(nullptr),
+  _allocator(nullptr),
   _verifier(nullptr),
   _phase_timings(nullptr),
   _monitoring_support(nullptr),
@@ -1030,7 +1033,7 @@ HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req
 
   // If TLAB request size is greater than available, allocate() will attempt to downsize request to fit within available
   // memory.
-  HeapWord* result = _free_set->allocate(req, in_new_region);
+  HeapWord* result = _allocator->allocate(req, in_new_region);
 
   // Record the plab configuration for this result and register the object.
   if (result != nullptr && req.is_old()) {

src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp

@@ -47,6 +47,7 @@
 
 class ConcurrentGCTimer;
 class ObjectIterateScanRootClosure;
+class ShenandoahAllocator;
 class ShenandoahCollectorPolicy;
 class ShenandoahGCSession;
 class ShenandoahGCStateResetter;
@@ -528,6 +529,7 @@ private:
   ShenandoahCollectorPolicy* _shenandoah_policy;
   ShenandoahMode*            _gc_mode;
   ShenandoahFreeSet*         _free_set;
+  ShenandoahAllocator*       _allocator;
   ShenandoahPacer*           _pacer;
   ShenandoahVerifier*        _verifier;
 
@@ -553,6 +555,7 @@ public:
   ShenandoahCollectorPolicy* shenandoah_policy() const { return _shenandoah_policy; }
   ShenandoahMode*            mode()              const { return _gc_mode;           }
   ShenandoahFreeSet*         free_set()          const { return _free_set;          }
+  ShenandoahAllocator*       allocator()         const { return _allocator;         }
   ShenandoahPacer*           pacer()             const { return _pacer;             }
 
   ShenandoahPhaseTimings*    phase_timings()     const { return _phase_timings;     }

src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.cpp

@@ -0,0 +1,411 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE THIS COPYRIGHT NOTICE OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "gc/shenandoah/shenandoahSerialAllocator.hpp"
+
+#include "gc/shenandoah/shenandoahAllocRequest.hpp"
+#include "gc/shenandoah/shenandoahFreeSet.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
+#include "gc/shared/cardTable.hpp"
+#include "gc/shared/plab.hpp"
+#include "logging/log.hpp"
+
+using idx_t = ShenandoahSimpleBitMap::idx_t;
+
+class ShenandoahLeftRightIterator {
+private:
+  idx_t _idx;
+  idx_t _end;
+  ShenandoahRegionPartitions* _partitions;
+  ShenandoahFreeSetPartitionId _partition;
+public:
+  explicit ShenandoahLeftRightIterator(ShenandoahRegionPartitions* partitions,
+                                       ShenandoahFreeSetPartitionId partition, bool use_empty = false)
+    : _idx(0), _end(0), _partitions(partitions), _partition(partition) {
+    _idx = use_empty ? _partitions->leftmost_empty(_partition) : _partitions->leftmost(_partition);
+    _end = use_empty ? _partitions->rightmost_empty(_partition) : _partitions->rightmost(_partition);
+  }
+
+  bool has_next() const {
+    if (_idx <= _end) {
+      assert(_partitions->in_free_set(_partition, _idx), "Boundaries or find_last_set_bit failed: %zd", _idx);
+      return true;
+    }
+    return false;
+  }
+
+  idx_t current() const {
+    return _idx;
+  }
+
+  idx_t next() {
+    _idx = _partitions->find_index_of_next_available_region(_partition, _idx + 1);
+    return current();
+  }
+};
+
+class ShenandoahRightLeftIterator {
+private:
+  idx_t _idx;
+  idx_t _end;
+  ShenandoahRegionPartitions* _partitions;
+  ShenandoahFreeSetPartitionId _partition;
+public:
+  explicit ShenandoahRightLeftIterator(ShenandoahRegionPartitions* partitions,
+                                       ShenandoahFreeSetPartitionId partition, bool use_empty = false)
+    : _idx(0), _end(0), _partitions(partitions), _partition(partition) {
+    _idx = use_empty ? _partitions->rightmost_empty(_partition) : _partitions->rightmost(_partition);
+    _end = use_empty ? _partitions->leftmost_empty(_partition) : _partitions->leftmost(_partition);
+  }
+
+  bool has_next() const {
+    if (_idx >= _end) {
+      assert(_partitions->in_free_set(_partition, _idx), "Boundaries or find_last_set_bit failed: %zd", _idx);
+      return true;
+    }
+    return false;
+  }
+
+  idx_t current() const {
+    return _idx;
+  }
+
+  idx_t next() {
+    _idx = _partitions->find_index_of_previous_available_region(_partition, _idx - 1);
+    return current();
+  }
+};
+
+ShenandoahSerialAllocator::ShenandoahSerialAllocator(ShenandoahFreeSet* free_set)
+  : ShenandoahAllocator(free_set),
+    _heap(ShenandoahHeap::heap()),
+    _alloc_bias_weight(INITIAL_ALLOC_BIAS_WEIGHT) {}
+
+HeapWord* ShenandoahSerialAllocator::allocate(ShenandoahAllocRequest& req, bool& in_new_region) {
+  shenandoah_assert_heaplocked();
+  if (ShenandoahHeapRegion::requires_humongous(req.size())) {
+    switch (req.type()) {
+      case ShenandoahAllocRequest::_alloc_shared:
+      case ShenandoahAllocRequest::_alloc_shared_gc:
+        in_new_region = true;
+        return _free_set->allocate_contiguous(req, /* is_humongous = */ true);
+      case ShenandoahAllocRequest::_alloc_cds:
+        in_new_region = true;
+        return _free_set->allocate_contiguous(req, /* is_humongous = */ false);
+      case ShenandoahAllocRequest::_alloc_plab:
+      case ShenandoahAllocRequest::_alloc_gclab:
+      case ShenandoahAllocRequest::_alloc_tlab:
+        in_new_region = false;
+        assert(false, "Trying to allocate TLAB in humongous region: %zu", req.size());
+        return nullptr;
+      default:
+        ShouldNotReachHere();
+        return nullptr;
+    }
+  } else {
+    return allocate_single(req, in_new_region);
+  }
+}
+
+HeapWord* ShenandoahSerialAllocator::allocate_single(ShenandoahAllocRequest& req, bool& in_new_region) {
+  shenandoah_assert_heaplocked();
+
+  if (req.is_mutator_alloc()) {
+    return allocate_for_mutator(req, in_new_region);
+  } else {
+    return allocate_for_collector(req, in_new_region);
+  }
+}
+
+HeapWord* ShenandoahSerialAllocator::allocate_for_mutator(ShenandoahAllocRequest& req, bool& in_new_region) {
+  update_allocation_bias();
+
+  ShenandoahRegionPartitions& partitions = _free_set->_partitions;
+  if (partitions.is_empty(ShenandoahFreeSetPartitionId::Mutator)) {
+    return nullptr;
+  }
+
+  if (partitions.alloc_from_left_bias(ShenandoahFreeSetPartitionId::Mutator)) {
+    ShenandoahLeftRightIterator iterator(&partitions, ShenandoahFreeSetPartitionId::Mutator);
+    return allocate_from_regions(iterator, req, in_new_region);
+  }
+
+  ShenandoahRightLeftIterator iterator(&partitions, ShenandoahFreeSetPartitionId::Mutator);
+  return allocate_from_regions(iterator, req, in_new_region);
+}
+
+// Alternating allocation direction between GC passes improves evacuation performance by
+// consuming partially-used regions before they become uncollectable floating garbage.
+// We bias toward the side with fewer non-empty regions to pack allocations tightly.
+void ShenandoahSerialAllocator::update_allocation_bias() {
+  if (_alloc_bias_weight-- <= 0) {
+    ShenandoahRegionPartitions& partitions = _free_set->_partitions;
+
+    idx_t non_empty_on_left = (partitions.leftmost_empty(ShenandoahFreeSetPartitionId::Mutator)
+                               - partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator));
+    idx_t non_empty_on_right = (partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator)
+                                - partitions.rightmost_empty(ShenandoahFreeSetPartitionId::Mutator));
+    partitions.set_bias_from_left_to_right(ShenandoahFreeSetPartitionId::Mutator, (non_empty_on_right < non_empty_on_left));
+    _alloc_bias_weight = INITIAL_ALLOC_BIAS_WEIGHT;
+  }
+}
+
+template<typename Iter>
+HeapWord* ShenandoahSerialAllocator::allocate_from_regions(Iter& iterator, ShenandoahAllocRequest& req, bool& in_new_region) {
+  for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) {
+    ShenandoahHeapRegion* r = _heap->get_region(idx);
+    size_t min_size = req.is_lab_alloc() ? req.min_size() : req.size();
+    if (_free_set->alloc_capacity(r) >= min_size * HeapWordSize) {
+      HeapWord* result = try_allocate_in(r, req, in_new_region);
+      if (result != nullptr) {
+        return result;
+      }
+    }
+  }
+  return nullptr;
+}
+
+// Collector allocation: first try the reserved Collector/OldCollector partition,
+// preferring regions with matching affiliation. If that fails and ShenandoahEvacReserveOverflow
+// is enabled, steal an empty region from the Mutator partition.
+HeapWord* ShenandoahSerialAllocator::allocate_for_collector(ShenandoahAllocRequest& req, bool& in_new_region) {
+  shenandoah_assert_heaplocked();
+  ShenandoahRegionPartitions& partitions = _free_set->_partitions;
+  ShenandoahFreeSetPartitionId which_partition = req.is_old() ? ShenandoahFreeSetPartitionId::OldCollector
+                                                              : ShenandoahFreeSetPartitionId::Collector;
+  HeapWord* result = nullptr;
+  if (partitions.alloc_from_left_bias(which_partition)) {
+    ShenandoahLeftRightIterator iterator(&partitions, which_partition);
+    result = allocate_with_affiliation(iterator, req.affiliation(), req, in_new_region);
+  } else {
+    ShenandoahRightLeftIterator iterator(&partitions, which_partition);
+    result = allocate_with_affiliation(iterator, req.affiliation(), req, in_new_region);
+  }
+
+  if (result != nullptr) {
+    return result;
+  }
+
+  if (!ShenandoahEvacReserveOverflow) {
+    return nullptr;
+  }
+
+  // Overflow: steal empty region from mutator partition for collector use.
+  if (partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::Mutator) > 0) {
+    result = try_allocate_from_mutator(req, in_new_region);
+  }
+
+  return result;
+}
+
+template<typename Iter>
+HeapWord* ShenandoahSerialAllocator::allocate_with_affiliation(Iter& iterator,
+                                                               ShenandoahAffiliation affiliation,
+                                                               ShenandoahAllocRequest& req,
+                                                               bool& in_new_region) {
+  assert(affiliation != ShenandoahAffiliation::FREE, "Must not");
+  ShenandoahHeapRegion* free_region = nullptr;
+  for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) {
+    ShenandoahHeapRegion* r = _heap->get_region(idx);
+    if (r->affiliation() == affiliation) {
+      HeapWord* result = try_allocate_in(r, req, in_new_region);
+      if (result != nullptr) {
+        return result;
+      }
+    } else if (free_region == nullptr && r->affiliation() == FREE) {
+      free_region = r;
+    }
+  }
+  if (free_region != nullptr) {
+    HeapWord* result = try_allocate_in(free_region, req, in_new_region);
+    assert(result != nullptr, "Allocate in free region in the partition always succeed.");
+    return result;
+  }
+  log_debug(gc, free)("Could not allocate collector region with affiliation: %s for request " PTR_FORMAT,
+                      shenandoah_affiliation_name(affiliation), p2i(&req));
+  return nullptr;
+}
+
+// Flip an empty region from Mutator to Collector/OldCollector partition, then allocate in it.
+// Searches from right to left to keep longer-lived collector regions at high addresses.
+HeapWord* ShenandoahSerialAllocator::try_allocate_from_mutator(ShenandoahAllocRequest& req, bool& in_new_region) {
+  ShenandoahRegionPartitions& partitions = _free_set->_partitions;
+  ShenandoahRightLeftIterator iterator(&partitions, ShenandoahFreeSetPartitionId::Mutator, true);
+  for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) {
+    ShenandoahHeapRegion* r = _heap->get_region(idx);
+    if (_free_set->can_allocate_from(r)) {
+      if (req.is_old()) {
+        if (!_free_set->flip_to_old_gc(r)) {
+          continue;
+        }
+      } else {
+        _free_set->flip_to_gc(r);
+      }
+      log_debug(gc, free)("Flipped region %zu to gc for request: " PTR_FORMAT, idx, p2i(&req));
+      return try_allocate_in(r, req, in_new_region);
+    }
+  }
+
+  return nullptr;
+}
+
+// Allocate within region r for the given request. This handles:
+// 1. Region recycling and affiliation setup for new (empty) regions
+// 2. LAB sizing (TLAB/GCLAB shrink-to-fit, PLAB card-alignment)
+// 3. Partition accounting (used, empty counts, retirement)
+HeapWord* ShenandoahSerialAllocator::try_allocate_in(ShenandoahHeapRegion* r, ShenandoahAllocRequest& req, bool& in_new_region) {
+  assert(_free_set->has_alloc_capacity(r), "Performance: should avoid full regions on this path: %zu", r->index());
+
+  ShenandoahHeap* heap = ShenandoahHeap::heap();
+  // Trash regions cannot be used while weak roots processing needs accurate marking info.
+  if (heap->is_concurrent_weak_root_in_progress() && r->is_trash()) {
+    return nullptr;
+  }
+
+  HeapWord* result = nullptr;
+  r->try_recycle_under_lock();
+  in_new_region = r->is_empty();
+  if (in_new_region) {
+    log_debug(gc, free)("Using new region (%zu) for %s (" PTR_FORMAT ").",
+                        r->index(), req.type_string(), p2i(&req));
+    assert(!r->is_affiliated(), "New region %zu should be unaffiliated", r->index());
+    r->set_affiliation(req.affiliation());
+    if (r->is_old()) {
+      r->end_preemptible_coalesce_and_fill();
+    }
+#ifdef ASSERT
+    ShenandoahMarkingContext* const ctx = heap->marking_context();
+    assert(ctx->top_at_mark_start(r) == r->bottom(), "Newly established allocation region starts with TAMS equal to bottom");
+    assert(ctx->is_bitmap_range_within_region_clear(ctx->top_bitmap(r), r->end()), "Bitmap above top_bitmap() must be clear");
+#endif
+  } else {
+    assert(r->is_affiliated(), "Region %zu that is not new should be affiliated", r->index());
+    if (r->affiliation() != req.affiliation()) {
+      assert(heap->mode()->is_generational(), "Request for %s from %s region should only happen in generational mode.",
+             req.affiliation_name(), r->affiliation_name());
+      return nullptr;
+    }
+  }
+
+  // Perform the actual allocation: LABs may be shrunk to fit, PLABs must be card-aligned.
+  if (req.is_lab_alloc()) {
+    size_t adjusted_size = req.size();
+    size_t free = r->free();
+    if (req.is_old()) {
+      assert(heap->mode()->is_generational(), "PLABs are only for generational mode");
+      assert(_free_set->_partitions.in_free_set(ShenandoahFreeSetPartitionId::OldCollector, r->index()),
+             "PLABS must be allocated in old_collector_free regions");
+
+      size_t usable_free = _free_set->get_usable_free_words(free);
+      if (adjusted_size > usable_free) {
+        adjusted_size = usable_free;
+      }
+      adjusted_size = align_down(adjusted_size, CardTable::card_size_in_words());
+      if (adjusted_size >= req.min_size()) {
+        result = _free_set->allocate_aligned_plab(adjusted_size, req, r);
+        assert(result != nullptr, "allocate must succeed");
+        req.set_actual_size(adjusted_size);
+      } else {
+        log_trace(gc, free)("Failed to shrink PLAB request (%zu) in region %zu to %zu"
+                            " because min_size() is %zu", req.size(), r->index(), adjusted_size, req.min_size());
+      }
+    } else {
+      free = align_down(free >> LogHeapWordSize, MinObjAlignment);
+      if (adjusted_size > free) {
+        adjusted_size = free;
+      }
+      if (adjusted_size >= req.min_size()) {
+        result = r->allocate(adjusted_size, req);
+        assert(result != nullptr, "Allocation must succeed: free %zu, actual %zu", free, adjusted_size);
+        req.set_actual_size(adjusted_size);
+      } else {
+        log_trace(gc, free)("Failed to shrink TLAB or GCLAB request (%zu) in region %zu to %zu"
+                            " because min_size() is %zu", req.size(), r->index(), adjusted_size, req.min_size());
+      }
+    }
+  } else {
+    size_t size = req.size();
+    result = r->allocate(size, req);
+    if (result != nullptr) {
+      req.set_actual_size(size);
+    }
+  }
+
+  // Update partition used bytes on success.
+  if (result != nullptr) {
+    if (req.is_mutator_alloc()) {
+      assert(req.is_young(), "Mutator allocations always come from young generation.");
+      _free_set->_partitions.increase_used(ShenandoahFreeSetPartitionId::Mutator, req.actual_size() * HeapWordSize);
+      _free_set->increase_bytes_allocated(req.actual_size() * HeapWordSize);
+    } else {
+      assert(req.is_gc_alloc(), "Should be gc_alloc since req wasn't mutator alloc");
+      // GC allocations set update_watermark so relocated objects aren't re-updated during update-refs.
+      r->set_update_watermark(r->top());
+      if (r->is_old()) {
+        _free_set->_partitions.increase_used(ShenandoahFreeSetPartitionId::OldCollector, (req.actual_size() + req.waste()) * HeapWordSize);
+      } else {
+        _free_set->_partitions.increase_used(ShenandoahFreeSetPartitionId::Collector, (req.actual_size() + req.waste()) * HeapWordSize);
+      }
+    }
+  }
+
+  ShenandoahFreeSetPartitionId orig_partition;
+  if (req.is_mutator_alloc()) {
+    orig_partition = ShenandoahFreeSetPartitionId::Mutator;
+  } else if (req.is_old()) {
+    orig_partition = ShenandoahFreeSetPartitionId::OldCollector;
+  } else {
+    orig_partition = ShenandoahFreeSetPartitionId::Collector;
+  }
+
+  DEBUG_ONLY(bool boundary_changed = false;)
+  if ((result != nullptr) && in_new_region) {
+    _free_set->_partitions.one_region_is_no_longer_empty(orig_partition);
+    DEBUG_ONLY(boundary_changed = true;)
+  }
+
+  // Retire the region if remaining capacity is too small for any future PLAB.
+  if (_free_set->alloc_capacity(r) < PLAB::min_size() * HeapWordSize) {
+    size_t idx = r->index();
+    size_t waste_bytes = _free_set->_partitions.retire_from_partition(orig_partition, idx, r->used());
+    DEBUG_ONLY(boundary_changed = true;)
+    if (req.is_mutator_alloc() && (waste_bytes > 0)) {
+      _free_set->increase_bytes_allocated(waste_bytes);
+    }
+  }
+
+  // Recompute generation used/affiliated totals.
+  _free_set->notify_allocation(orig_partition, in_new_region);
+
+#ifdef ASSERT
+  if (boundary_changed) {
+    _free_set->_partitions.assert_bounds();
+  } else {
+    _free_set->_partitions.assert_bounds_sanity();
+  }
+#endif
+  return result;
+}

src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.hpp

@@ -0,0 +1,82 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE THIS COPYRIGHT NOTICE OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHSERIALALLOCATOR_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHSERIALALLOCATOR_HPP
+
+#include "gc/shenandoah/shenandoahAllocator.hpp"
+#include "gc/shenandoah/shenandoahAffiliation.hpp"
+
+class ShenandoahFreeSet;
+class ShenandoahHeap;
+class ShenandoahHeapRegion;
+class ShenandoahRegionPartitions;
+
+// ShenandoahSerialAllocator performs all allocations serially under the heap lock.
+// It selects regions from the partitioned free set:
+//  - Mutator allocations: from Mutator partition with left/right bias alternation
+//  - Collector allocations: from Collector/OldCollector partition with affiliation preference
+//  - Overflow: collector may steal empty regions from Mutator partition
+//  - Humongous: delegated to ShenandoahFreeSet::allocate_contiguous()
+class ShenandoahSerialAllocator : public ShenandoahAllocator {
+private:
+  ShenandoahHeap* const _heap;
+
+  // Allocation direction alternates to avoid repeatedly skipping the same uncollected regions.
+  ssize_t _alloc_bias_weight;
+  static const ssize_t INITIAL_ALLOC_BIAS_WEIGHT = 256;
+
+  // Attempt allocation within a single region. Handles LAB sizing, PLAB card-alignment,
+  // affiliation checks, and updates partition accounting via ShenandoahFreeSet.
+  // Retires the region if remaining capacity falls below PLAB::min_size().
+  HeapWord* try_allocate_in(ShenandoahHeapRegion* r, ShenandoahAllocRequest& req, bool& in_new_region);
+
+  HeapWord* allocate_single(ShenandoahAllocRequest& req, bool& in_new_region);
+  HeapWord* allocate_for_mutator(ShenandoahAllocRequest& req, bool& in_new_region);
+  HeapWord* allocate_for_collector(ShenandoahAllocRequest& req, bool& in_new_region);
+
+  // Steal an empty region from Mutator partition for collector use.
+  HeapWord* try_allocate_from_mutator(ShenandoahAllocRequest& req, bool& in_new_region);
+
+  // Re-evaluate left-to-right vs right-to-left bias for mutator allocations.
+  void update_allocation_bias();
+
+  // Scan regions using the given iterator, attempt allocation in each.
+  template<typename Iter>
+  HeapWord* allocate_from_regions(Iter& iterator, ShenandoahAllocRequest& req, bool& in_new_region);
+
+  // For collector: prefer regions matching the requested affiliation, fall back to FREE regions.
+  template<typename Iter>
+  HeapWord* allocate_with_affiliation(Iter& iterator,
+                                      ShenandoahAffiliation affiliation,
+                                      ShenandoahAllocRequest& req,
+                                      bool& in_new_region);
+
+public:
+  ShenandoahSerialAllocator(ShenandoahFreeSet* free_set);
+
+  HeapWord* allocate(ShenandoahAllocRequest& req, bool& in_new_region) override;
+};
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHSERIALALLOCATOR_HPP