From 4c2da56ab1bfab89107b13b8581a68d0d6c6afbc Mon Sep 17 00:00:00 2001 From: Xiaolong Peng Date: Thu, 28 May 2026 12:36:50 -0700 Subject: [PATCH] 8385592: Shenandoah: Encapsulate memory allocation path into ShenandoahAllocator interface --- .../gc/shenandoah/shenandoahAllocator.hpp | 49 +++ .../share/gc/shenandoah/shenandoahFreeSet.cpp | 63 ++- .../share/gc/shenandoah/shenandoahFreeSet.hpp | 35 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 5 +- .../share/gc/shenandoah/shenandoahHeap.hpp | 3 + .../shenandoah/shenandoahSerialAllocator.cpp | 411 ++++++++++++++++++ .../shenandoah/shenandoahSerialAllocator.hpp | 82 ++++ 7 files changed, 623 insertions(+), 25 deletions(-) create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahAllocator.hpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.cpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.hpp diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAllocator.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAllocator.hpp new file mode 100644 index 00000000000..4ebf2f8c40e --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahAllocator.hpp @@ -0,0 +1,49 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE THIS COPYRIGHT NOTICE OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHALLOCATOR_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHALLOCATOR_HPP + +#include "memory/allocation.hpp" + +class ShenandoahAllocRequest; +class ShenandoahFreeSet; + +// ShenandoahAllocator defines the allocation interface for the Shenandoah GC. +// Subclasses implement different allocation strategies (e.g. serial under heap lock, CAS-based). +// Partition accounting is delegated back to ShenandoahFreeSet. +class ShenandoahAllocator : public CHeapObj { +protected: + ShenandoahFreeSet* const _free_set; + +public: + ShenandoahAllocator(ShenandoahFreeSet* free_set) : _free_set(free_set) {} + virtual ~ShenandoahAllocator() = default; + + // Allocate memory for the given request. Returns nullptr on failure. + // Sets in_new_region to true if allocation consumes a previously empty region. + virtual HeapWord* allocate(ShenandoahAllocRequest& req, bool& in_new_region) = 0; +}; + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHALLOCATOR_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index fdce385e0f6..b9ca9605cc9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -222,18 +222,18 @@ ShenandoahFreeSetPartitionId ShenandoahFreeSet::prepare_to_promote_in_place(size return p; } -inline bool ShenandoahFreeSet::can_allocate_from(ShenandoahHeapRegion *r) const { +bool ShenandoahFreeSet::can_allocate_from(ShenandoahHeapRegion *r) const { const auto state = r->state(); return ShenandoahHeapRegion::is_empty_state(state) || (ShenandoahHeapRegion::is_trash(state) && !_heap->is_concurrent_weak_root_in_progress()); } -inline bool ShenandoahFreeSet::can_allocate_from(size_t idx) const { +bool ShenandoahFreeSet::can_allocate_from(size_t idx) const { ShenandoahHeapRegion* r = _heap->get_region(idx); return can_allocate_from(r); } -inline size_t ShenandoahFreeSet::alloc_capacity(ShenandoahHeapRegion *r) const { +size_t ShenandoahFreeSet::alloc_capacity(ShenandoahHeapRegion *r) const { if (r->is_trash()) { // This would be recycled on allocation path return ShenandoahHeapRegion::region_size_bytes(); @@ -242,12 +242,12 @@ inline size_t ShenandoahFreeSet::alloc_capacity(ShenandoahHeapRegion *r) const { } } -inline size_t ShenandoahFreeSet::alloc_capacity(size_t idx) const { +size_t ShenandoahFreeSet::alloc_capacity(size_t idx) const { ShenandoahHeapRegion* r = _heap->get_region(idx); return alloc_capacity(r); } -inline bool ShenandoahFreeSet::has_alloc_capacity(ShenandoahHeapRegion *r) const { +bool ShenandoahFreeSet::has_alloc_capacity(ShenandoahHeapRegion *r) const { return alloc_capacity(r) > 0; } @@ -315,7 +315,48 @@ void ShenandoahFreeSet::increase_bytes_allocated(size_t bytes) { _mutator_bytes_allocated_since_gc_start += bytes; } -inline idx_t ShenandoahRegionPartitions::leftmost(ShenandoahFreeSetPartitionId which_partition) const { +void ShenandoahFreeSet::notify_allocation(ShenandoahFreeSetPartitionId partition, bool in_new_region) { + switch (partition) { + case ShenandoahFreeSetPartitionId::Mutator: + recompute_total_used(); + if (in_new_region) { + recompute_total_affiliated(); + } + break; + case ShenandoahFreeSetPartitionId::Collector: + recompute_total_used(); + if (in_new_region) { + recompute_total_affiliated(); + } + break; + case ShenandoahFreeSetPartitionId::OldCollector: + recompute_total_used(); + if (in_new_region) { + recompute_total_affiliated(); + } + break; + case ShenandoahFreeSetPartitionId::NotFree: + default: + assert(false, "won't happen"); + } +} + +idx_t ShenandoahRegionPartitions::leftmost(ShenandoahFreeSetPartitionId which_partition) const { assert (which_partition < NumPartitions, "selected free partition must be valid"); idx_t idx = _leftmosts[int(which_partition)]; if (idx >= _max) { @@ -328,7 +369,7 @@ inline idx_t ShenandoahRegionPartitions::leftmost(ShenandoahFreeSetPartitionId w } } -inline idx_t ShenandoahRegionPartitions::rightmost(ShenandoahFreeSetPartitionId which_partition) const { +idx_t ShenandoahRegionPartitions::rightmost(ShenandoahFreeSetPartitionId which_partition) const { assert (which_partition < NumPartitions, "selected free partition must be valid"); idx_t idx = _rightmosts[int(which_partition)]; // Cannot assert that membership[which_partition.is_set(idx) because this helper method may be used @@ -819,12 +860,12 @@ inline bool ShenandoahRegionPartitions::partition_id_matches(idx_t idx, Shenando } #endif -inline bool ShenandoahRegionPartitions::is_empty(ShenandoahFreeSetPartitionId which_partition) const { +bool ShenandoahRegionPartitions::is_empty(ShenandoahFreeSetPartitionId which_partition) const { assert (which_partition < NumPartitions, "selected free partition must be valid"); return (leftmost(which_partition) > rightmost(which_partition)); } -inline idx_t ShenandoahRegionPartitions::find_index_of_next_available_region( +idx_t ShenandoahRegionPartitions::find_index_of_next_available_region( ShenandoahFreeSetPartitionId which_partition, idx_t start_index) const { idx_t rightmost_idx = rightmost(which_partition); idx_t leftmost_idx = leftmost(which_partition); @@ -840,7 +881,7 @@ inline idx_t ShenandoahRegionPartitions::find_index_of_next_available_region( return result; } -inline idx_t ShenandoahRegionPartitions::find_index_of_previous_available_region( +idx_t ShenandoahRegionPartitions::find_index_of_previous_available_region( ShenandoahFreeSetPartitionId which_partition, idx_t last_index) const { idx_t rightmost_idx = rightmost(which_partition); idx_t leftmost_idx = leftmost(which_partition); @@ -1213,7 +1254,7 @@ void ShenandoahRegionPartitions::assert_bounds() { "Mutator humongous waste must match"); } -inline void ShenandoahRegionPartitions::assert_bounds_sanity() { +void ShenandoahRegionPartitions::assert_bounds_sanity() { for (uint8_t i = 0; i < UIntNumPartitions; i++) { ShenandoahFreeSetPartitionId partition = static_cast(i); assert(leftmost(partition) == _max || membership(leftmost(partition)) == partition, "Left most boundry must be sane"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp index f7ba1f05f47..93fcaf08f89 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp @@ -35,6 +35,9 @@ typedef ShenandoahLock ShenandoahRebuildLock; typedef ShenandoahLocker ShenandoahRebuildLocker; +class ShenandoahAllocator; +class ShenandoahSerialAllocator; + // Each ShenandoahHeapRegion is associated with a ShenandoahFreeSetPartitionId. enum class ShenandoahFreeSetPartitionId : uint8_t { Mutator, // Region is in the Mutator free set: available memory is available to mutators. @@ -161,7 +164,7 @@ public: _membership[int(p)].clear_bit(idx); } - inline void one_region_is_no_longer_empty(ShenandoahFreeSetPartitionId partition); + void one_region_is_no_longer_empty(ShenandoahFreeSetPartitionId partition); // Set the Mutator intervals, usage, and capacity according to arguments. Reset the Collector intervals, used, capacity // to represent empty Collector free set. We use this at the end of rebuild_free_set() to avoid the overhead of making @@ -231,11 +234,11 @@ public: const char* partition_membership_name(idx_t idx) const; // Return the index of the next available region >= start_index, or maximum_regions if not found. - inline idx_t find_index_of_next_available_region(ShenandoahFreeSetPartitionId which_partition, + idx_t find_index_of_next_available_region(ShenandoahFreeSetPartitionId which_partition, idx_t start_index) const; // Return the index of the previous available region <= last_index, or -1 if not found. - inline idx_t find_index_of_previous_available_region(ShenandoahFreeSetPartitionId which_partition, + idx_t find_index_of_previous_available_region(ShenandoahFreeSetPartitionId which_partition, idx_t last_index) const; // Return the index of the next available cluster of cluster_size regions >= start_index, or maximum_regions if not found. @@ -279,12 +282,12 @@ public: // leftmost() and leftmost_empty() return _max, rightmost() and rightmost_empty() return 0 // otherwise, expect the following: // 0 <= leftmost <= leftmost_empty <= rightmost_empty <= rightmost < _max - inline idx_t leftmost(ShenandoahFreeSetPartitionId which_partition) const; - inline idx_t rightmost(ShenandoahFreeSetPartitionId which_partition) const; + idx_t leftmost(ShenandoahFreeSetPartitionId which_partition) const; + idx_t rightmost(ShenandoahFreeSetPartitionId which_partition) const; idx_t leftmost_empty(ShenandoahFreeSetPartitionId which_partition); idx_t rightmost_empty(ShenandoahFreeSetPartitionId which_partition); - inline bool is_empty(ShenandoahFreeSetPartitionId which_partition) const; + bool is_empty(ShenandoahFreeSetPartitionId which_partition) const; inline void increase_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions); inline void decrease_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions); @@ -311,7 +314,7 @@ public: inline void decrease_available(ShenandoahFreeSetPartitionId which_partition, size_t bytes); inline size_t get_available(ShenandoahFreeSetPartitionId which_partition); - inline void increase_used(ShenandoahFreeSetPartitionId which_partition, size_t bytes); + void increase_used(ShenandoahFreeSetPartitionId which_partition, size_t bytes); inline void decrease_used(ShenandoahFreeSetPartitionId which_partition, size_t bytes); inline size_t get_used(ShenandoahFreeSetPartitionId which_partition) { assert (which_partition < NumPartitions, "Partition must be valid"); @@ -404,7 +407,7 @@ public: void assert_bounds() NOT_DEBUG_RETURN; // this checks certain sanity conditions related to the bounds with much less effort than is required to // more rigorously enforce correctness as is done by assert_bounds() - inline void assert_bounds_sanity() NOT_DEBUG_RETURN; + void assert_bounds_sanity() NOT_DEBUG_RETURN; }; // Publicly, ShenandoahFreeSet represents memory that is available to mutator threads. The public capacity(), used(), @@ -432,6 +435,9 @@ public: // during the next GC pass. class ShenandoahFreeSet : public CHeapObj { + friend class ShenandoahAllocator; + friend class ShenandoahSerialAllocator; + using idx_t = ShenandoahSimpleBitMap::idx_t; private: ShenandoahHeap* const _heap; @@ -622,10 +628,10 @@ private: // Returns true iff this region is entirely available, either because it is empty() or because it has been found to represent // immediate trash and we'll be able to immediately recycle it. Note that we cannot recycle immediate trash if // concurrent weak root processing is in progress. - inline bool can_allocate_from(ShenandoahHeapRegion *r) const; - inline bool can_allocate_from(size_t idx) const; + bool can_allocate_from(ShenandoahHeapRegion *r) const; + bool can_allocate_from(size_t idx) const; - inline bool has_alloc_capacity(ShenandoahHeapRegion *r) const; + bool has_alloc_capacity(ShenandoahHeapRegion *r) const; void transfer_empty_regions_from_to(ShenandoahFreeSetPartitionId source_partition, ShenandoahFreeSetPartitionId dest_partition, @@ -668,6 +674,9 @@ public: void increase_bytes_allocated(size_t bytes); + // Called by ShenandoahAllocator after a successful allocation to update used/affiliated totals. + void notify_allocation(ShenandoahFreeSetPartitionId partition, bool in_new_region); + // Return an approximation of the bytes allocated since GC start. The value returned is monotonically non-decreasing // in time within each GC cycle. For certain GC cycles, the value returned may include some bytes allocated before // the start of the current GC cycle. @@ -696,8 +705,8 @@ public: } // Public because ShenandoahRegionPartitions assertions require access. - inline size_t alloc_capacity(ShenandoahHeapRegion *r) const; - inline size_t alloc_capacity(size_t idx) const; + size_t alloc_capacity(ShenandoahHeapRegion *r) const; + size_t alloc_capacity(size_t idx) const; // Return bytes used by old inline size_t old_used() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 2bedc53e24b..f369138b3fb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -43,6 +43,7 @@ #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp" #include "gc/shenandoah/mode/shenandoahSATBMode.hpp" #include "gc/shenandoah/shenandoahAllocRequest.hpp" +#include "gc/shenandoah/shenandoahSerialAllocator.hpp" #include "gc/shenandoah/shenandoahBarrierSet.hpp" #include "gc/shenandoah/shenandoahClosures.inline.hpp" #include "gc/shenandoah/shenandoahCodeRoots.hpp" @@ -436,6 +437,7 @@ jint ShenandoahHeap::initialize() { } _free_set = new ShenandoahFreeSet(this, _num_regions); + _allocator = new ShenandoahSerialAllocator(_free_set); initialize_generations(); // We are initializing free set. We ignore cset region tallies. @@ -576,6 +578,7 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _shenandoah_policy(policy), _gc_mode(nullptr), _free_set(nullptr), + _allocator(nullptr), _verifier(nullptr), _phase_timings(nullptr), _monitoring_support(nullptr), @@ -1030,7 +1033,7 @@ HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req // If TLAB request size is greater than available, allocate() will attempt to downsize request to fit within available // memory. - HeapWord* result = _free_set->allocate(req, in_new_region); + HeapWord* result = _allocator->allocate(req, in_new_region); // Record the plab configuration for this result and register the object. if (result != nullptr && req.is_old()) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index fa435eaa1be..20d7dc3d171 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -47,6 +47,7 @@ class ConcurrentGCTimer; class ObjectIterateScanRootClosure; +class ShenandoahAllocator; class ShenandoahCollectorPolicy; class ShenandoahGCSession; class ShenandoahGCStateResetter; @@ -528,6 +529,7 @@ private: ShenandoahCollectorPolicy* _shenandoah_policy; ShenandoahMode* _gc_mode; ShenandoahFreeSet* _free_set; + ShenandoahAllocator* _allocator; ShenandoahPacer* _pacer; ShenandoahVerifier* _verifier; @@ -553,6 +555,7 @@ public: ShenandoahCollectorPolicy* shenandoah_policy() const { return _shenandoah_policy; } ShenandoahMode* mode() const { return _gc_mode; } ShenandoahFreeSet* free_set() const { return _free_set; } + ShenandoahAllocator* allocator() const { return _allocator; } ShenandoahPacer* pacer() const { return _pacer; } ShenandoahPhaseTimings* phase_timings() const { return _phase_timings; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.cpp b/src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.cpp new file mode 100644 index 00000000000..9051970003e --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.cpp @@ -0,0 +1,411 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE THIS COPYRIGHT NOTICE OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "gc/shenandoah/shenandoahSerialAllocator.hpp" + +#include "gc/shenandoah/shenandoahAllocRequest.hpp" +#include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" +#include "gc/shared/cardTable.hpp" +#include "gc/shared/plab.hpp" +#include "logging/log.hpp" + +using idx_t = ShenandoahSimpleBitMap::idx_t; + +class ShenandoahLeftRightIterator { +private: + idx_t _idx; + idx_t _end; + ShenandoahRegionPartitions* _partitions; + ShenandoahFreeSetPartitionId _partition; +public: + explicit ShenandoahLeftRightIterator(ShenandoahRegionPartitions* partitions, + ShenandoahFreeSetPartitionId partition, bool use_empty = false) + : _idx(0), _end(0), _partitions(partitions), _partition(partition) { + _idx = use_empty ? _partitions->leftmost_empty(_partition) : _partitions->leftmost(_partition); + _end = use_empty ? _partitions->rightmost_empty(_partition) : _partitions->rightmost(_partition); + } + + bool has_next() const { + if (_idx <= _end) { + assert(_partitions->in_free_set(_partition, _idx), "Boundaries or find_last_set_bit failed: %zd", _idx); + return true; + } + return false; + } + + idx_t current() const { + return _idx; + } + + idx_t next() { + _idx = _partitions->find_index_of_next_available_region(_partition, _idx + 1); + return current(); + } +}; + +class ShenandoahRightLeftIterator { +private: + idx_t _idx; + idx_t _end; + ShenandoahRegionPartitions* _partitions; + ShenandoahFreeSetPartitionId _partition; +public: + explicit ShenandoahRightLeftIterator(ShenandoahRegionPartitions* partitions, + ShenandoahFreeSetPartitionId partition, bool use_empty = false) + : _idx(0), _end(0), _partitions(partitions), _partition(partition) { + _idx = use_empty ? _partitions->rightmost_empty(_partition) : _partitions->rightmost(_partition); + _end = use_empty ? _partitions->leftmost_empty(_partition) : _partitions->leftmost(_partition); + } + + bool has_next() const { + if (_idx >= _end) { + assert(_partitions->in_free_set(_partition, _idx), "Boundaries or find_last_set_bit failed: %zd", _idx); + return true; + } + return false; + } + + idx_t current() const { + return _idx; + } + + idx_t next() { + _idx = _partitions->find_index_of_previous_available_region(_partition, _idx - 1); + return current(); + } +}; + +ShenandoahSerialAllocator::ShenandoahSerialAllocator(ShenandoahFreeSet* free_set) + : ShenandoahAllocator(free_set), + _heap(ShenandoahHeap::heap()), + _alloc_bias_weight(INITIAL_ALLOC_BIAS_WEIGHT) {} + +HeapWord* ShenandoahSerialAllocator::allocate(ShenandoahAllocRequest& req, bool& in_new_region) { + shenandoah_assert_heaplocked(); + if (ShenandoahHeapRegion::requires_humongous(req.size())) { + switch (req.type()) { + case ShenandoahAllocRequest::_alloc_shared: + case ShenandoahAllocRequest::_alloc_shared_gc: + in_new_region = true; + return _free_set->allocate_contiguous(req, /* is_humongous = */ true); + case ShenandoahAllocRequest::_alloc_cds: + in_new_region = true; + return _free_set->allocate_contiguous(req, /* is_humongous = */ false); + case ShenandoahAllocRequest::_alloc_plab: + case ShenandoahAllocRequest::_alloc_gclab: + case ShenandoahAllocRequest::_alloc_tlab: + in_new_region = false; + assert(false, "Trying to allocate TLAB in humongous region: %zu", req.size()); + return nullptr; + default: + ShouldNotReachHere(); + return nullptr; + } + } else { + return allocate_single(req, in_new_region); + } +} + +HeapWord* ShenandoahSerialAllocator::allocate_single(ShenandoahAllocRequest& req, bool& in_new_region) { + shenandoah_assert_heaplocked(); + + if (req.is_mutator_alloc()) { + return allocate_for_mutator(req, in_new_region); + } else { + return allocate_for_collector(req, in_new_region); + } +} + +HeapWord* ShenandoahSerialAllocator::allocate_for_mutator(ShenandoahAllocRequest& req, bool& in_new_region) { + update_allocation_bias(); + + ShenandoahRegionPartitions& partitions = _free_set->_partitions; + if (partitions.is_empty(ShenandoahFreeSetPartitionId::Mutator)) { + return nullptr; + } + + if (partitions.alloc_from_left_bias(ShenandoahFreeSetPartitionId::Mutator)) { + ShenandoahLeftRightIterator iterator(&partitions, ShenandoahFreeSetPartitionId::Mutator); + return allocate_from_regions(iterator, req, in_new_region); + } + + ShenandoahRightLeftIterator iterator(&partitions, ShenandoahFreeSetPartitionId::Mutator); + return allocate_from_regions(iterator, req, in_new_region); +} + +// Alternating allocation direction between GC passes improves evacuation performance by +// consuming partially-used regions before they become uncollectable floating garbage. +// We bias toward the side with fewer non-empty regions to pack allocations tightly. +void ShenandoahSerialAllocator::update_allocation_bias() { + if (_alloc_bias_weight-- <= 0) { + ShenandoahRegionPartitions& partitions = _free_set->_partitions; + + idx_t non_empty_on_left = (partitions.leftmost_empty(ShenandoahFreeSetPartitionId::Mutator) + - partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator)); + idx_t non_empty_on_right = (partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator) + - partitions.rightmost_empty(ShenandoahFreeSetPartitionId::Mutator)); + partitions.set_bias_from_left_to_right(ShenandoahFreeSetPartitionId::Mutator, (non_empty_on_right < non_empty_on_left)); + _alloc_bias_weight = INITIAL_ALLOC_BIAS_WEIGHT; + } +} + +template +HeapWord* ShenandoahSerialAllocator::allocate_from_regions(Iter& iterator, ShenandoahAllocRequest& req, bool& in_new_region) { + for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) { + ShenandoahHeapRegion* r = _heap->get_region(idx); + size_t min_size = req.is_lab_alloc() ? req.min_size() : req.size(); + if (_free_set->alloc_capacity(r) >= min_size * HeapWordSize) { + HeapWord* result = try_allocate_in(r, req, in_new_region); + if (result != nullptr) { + return result; + } + } + } + return nullptr; +} + +// Collector allocation: first try the reserved Collector/OldCollector partition, +// preferring regions with matching affiliation. If that fails and ShenandoahEvacReserveOverflow +// is enabled, steal an empty region from the Mutator partition. +HeapWord* ShenandoahSerialAllocator::allocate_for_collector(ShenandoahAllocRequest& req, bool& in_new_region) { + shenandoah_assert_heaplocked(); + ShenandoahRegionPartitions& partitions = _free_set->_partitions; + ShenandoahFreeSetPartitionId which_partition = req.is_old() ? ShenandoahFreeSetPartitionId::OldCollector + : ShenandoahFreeSetPartitionId::Collector; + HeapWord* result = nullptr; + if (partitions.alloc_from_left_bias(which_partition)) { + ShenandoahLeftRightIterator iterator(&partitions, which_partition); + result = allocate_with_affiliation(iterator, req.affiliation(), req, in_new_region); + } else { + ShenandoahRightLeftIterator iterator(&partitions, which_partition); + result = allocate_with_affiliation(iterator, req.affiliation(), req, in_new_region); + } + + if (result != nullptr) { + return result; + } + + if (!ShenandoahEvacReserveOverflow) { + return nullptr; + } + + // Overflow: steal empty region from mutator partition for collector use. + if (partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::Mutator) > 0) { + result = try_allocate_from_mutator(req, in_new_region); + } + + return result; +} + +template +HeapWord* ShenandoahSerialAllocator::allocate_with_affiliation(Iter& iterator, + ShenandoahAffiliation affiliation, + ShenandoahAllocRequest& req, + bool& in_new_region) { + assert(affiliation != ShenandoahAffiliation::FREE, "Must not"); + ShenandoahHeapRegion* free_region = nullptr; + for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) { + ShenandoahHeapRegion* r = _heap->get_region(idx); + if (r->affiliation() == affiliation) { + HeapWord* result = try_allocate_in(r, req, in_new_region); + if (result != nullptr) { + return result; + } + } else if (free_region == nullptr && r->affiliation() == FREE) { + free_region = r; + } + } + if (free_region != nullptr) { + HeapWord* result = try_allocate_in(free_region, req, in_new_region); + assert(result != nullptr, "Allocate in free region in the partition always succeed."); + return result; + } + log_debug(gc, free)("Could not allocate collector region with affiliation: %s for request " PTR_FORMAT, + shenandoah_affiliation_name(affiliation), p2i(&req)); + return nullptr; +} + +// Flip an empty region from Mutator to Collector/OldCollector partition, then allocate in it. +// Searches from right to left to keep longer-lived collector regions at high addresses. +HeapWord* ShenandoahSerialAllocator::try_allocate_from_mutator(ShenandoahAllocRequest& req, bool& in_new_region) { + ShenandoahRegionPartitions& partitions = _free_set->_partitions; + ShenandoahRightLeftIterator iterator(&partitions, ShenandoahFreeSetPartitionId::Mutator, true); + for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) { + ShenandoahHeapRegion* r = _heap->get_region(idx); + if (_free_set->can_allocate_from(r)) { + if (req.is_old()) { + if (!_free_set->flip_to_old_gc(r)) { + continue; + } + } else { + _free_set->flip_to_gc(r); + } + log_debug(gc, free)("Flipped region %zu to gc for request: " PTR_FORMAT, idx, p2i(&req)); + return try_allocate_in(r, req, in_new_region); + } + } + + return nullptr; +} + +// Allocate within region r for the given request. This handles: +// 1. Region recycling and affiliation setup for new (empty) regions +// 2. LAB sizing (TLAB/GCLAB shrink-to-fit, PLAB card-alignment) +// 3. Partition accounting (used, empty counts, retirement) +HeapWord* ShenandoahSerialAllocator::try_allocate_in(ShenandoahHeapRegion* r, ShenandoahAllocRequest& req, bool& in_new_region) { + assert(_free_set->has_alloc_capacity(r), "Performance: should avoid full regions on this path: %zu", r->index()); + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + // Trash regions cannot be used while weak roots processing needs accurate marking info. + if (heap->is_concurrent_weak_root_in_progress() && r->is_trash()) { + return nullptr; + } + + HeapWord* result = nullptr; + r->try_recycle_under_lock(); + in_new_region = r->is_empty(); + if (in_new_region) { + log_debug(gc, free)("Using new region (%zu) for %s (" PTR_FORMAT ").", + r->index(), req.type_string(), p2i(&req)); + assert(!r->is_affiliated(), "New region %zu should be unaffiliated", r->index()); + r->set_affiliation(req.affiliation()); + if (r->is_old()) { + r->end_preemptible_coalesce_and_fill(); + } +#ifdef ASSERT + ShenandoahMarkingContext* const ctx = heap->marking_context(); + assert(ctx->top_at_mark_start(r) == r->bottom(), "Newly established allocation region starts with TAMS equal to bottom"); + assert(ctx->is_bitmap_range_within_region_clear(ctx->top_bitmap(r), r->end()), "Bitmap above top_bitmap() must be clear"); +#endif + } else { + assert(r->is_affiliated(), "Region %zu that is not new should be affiliated", r->index()); + if (r->affiliation() != req.affiliation()) { + assert(heap->mode()->is_generational(), "Request for %s from %s region should only happen in generational mode.", + req.affiliation_name(), r->affiliation_name()); + return nullptr; + } + } + + // Perform the actual allocation: LABs may be shrunk to fit, PLABs must be card-aligned. + if (req.is_lab_alloc()) { + size_t adjusted_size = req.size(); + size_t free = r->free(); + if (req.is_old()) { + assert(heap->mode()->is_generational(), "PLABs are only for generational mode"); + assert(_free_set->_partitions.in_free_set(ShenandoahFreeSetPartitionId::OldCollector, r->index()), + "PLABS must be allocated in old_collector_free regions"); + + size_t usable_free = _free_set->get_usable_free_words(free); + if (adjusted_size > usable_free) { + adjusted_size = usable_free; + } + adjusted_size = align_down(adjusted_size, CardTable::card_size_in_words()); + if (adjusted_size >= req.min_size()) { + result = _free_set->allocate_aligned_plab(adjusted_size, req, r); + assert(result != nullptr, "allocate must succeed"); + req.set_actual_size(adjusted_size); + } else { + log_trace(gc, free)("Failed to shrink PLAB request (%zu) in region %zu to %zu" + " because min_size() is %zu", req.size(), r->index(), adjusted_size, req.min_size()); + } + } else { + free = align_down(free >> LogHeapWordSize, MinObjAlignment); + if (adjusted_size > free) { + adjusted_size = free; + } + if (adjusted_size >= req.min_size()) { + result = r->allocate(adjusted_size, req); + assert(result != nullptr, "Allocation must succeed: free %zu, actual %zu", free, adjusted_size); + req.set_actual_size(adjusted_size); + } else { + log_trace(gc, free)("Failed to shrink TLAB or GCLAB request (%zu) in region %zu to %zu" + " because min_size() is %zu", req.size(), r->index(), adjusted_size, req.min_size()); + } + } + } else { + size_t size = req.size(); + result = r->allocate(size, req); + if (result != nullptr) { + req.set_actual_size(size); + } + } + + // Update partition used bytes on success. + if (result != nullptr) { + if (req.is_mutator_alloc()) { + assert(req.is_young(), "Mutator allocations always come from young generation."); + _free_set->_partitions.increase_used(ShenandoahFreeSetPartitionId::Mutator, req.actual_size() * HeapWordSize); + _free_set->increase_bytes_allocated(req.actual_size() * HeapWordSize); + } else { + assert(req.is_gc_alloc(), "Should be gc_alloc since req wasn't mutator alloc"); + // GC allocations set update_watermark so relocated objects aren't re-updated during update-refs. + r->set_update_watermark(r->top()); + if (r->is_old()) { + _free_set->_partitions.increase_used(ShenandoahFreeSetPartitionId::OldCollector, (req.actual_size() + req.waste()) * HeapWordSize); + } else { + _free_set->_partitions.increase_used(ShenandoahFreeSetPartitionId::Collector, (req.actual_size() + req.waste()) * HeapWordSize); + } + } + } + + ShenandoahFreeSetPartitionId orig_partition; + if (req.is_mutator_alloc()) { + orig_partition = ShenandoahFreeSetPartitionId::Mutator; + } else if (req.is_old()) { + orig_partition = ShenandoahFreeSetPartitionId::OldCollector; + } else { + orig_partition = ShenandoahFreeSetPartitionId::Collector; + } + + DEBUG_ONLY(bool boundary_changed = false;) + if ((result != nullptr) && in_new_region) { + _free_set->_partitions.one_region_is_no_longer_empty(orig_partition); + DEBUG_ONLY(boundary_changed = true;) + } + + // Retire the region if remaining capacity is too small for any future PLAB. + if (_free_set->alloc_capacity(r) < PLAB::min_size() * HeapWordSize) { + size_t idx = r->index(); + size_t waste_bytes = _free_set->_partitions.retire_from_partition(orig_partition, idx, r->used()); + DEBUG_ONLY(boundary_changed = true;) + if (req.is_mutator_alloc() && (waste_bytes > 0)) { + _free_set->increase_bytes_allocated(waste_bytes); + } + } + + // Recompute generation used/affiliated totals. + _free_set->notify_allocation(orig_partition, in_new_region); + +#ifdef ASSERT + if (boundary_changed) { + _free_set->_partitions.assert_bounds(); + } else { + _free_set->_partitions.assert_bounds_sanity(); + } +#endif + return result; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.hpp b/src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.hpp new file mode 100644 index 00000000000..621da56b828 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.hpp @@ -0,0 +1,82 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE THIS COPYRIGHT NOTICE OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHSERIALALLOCATOR_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHSERIALALLOCATOR_HPP + +#include "gc/shenandoah/shenandoahAllocator.hpp" +#include "gc/shenandoah/shenandoahAffiliation.hpp" + +class ShenandoahFreeSet; +class ShenandoahHeap; +class ShenandoahHeapRegion; +class ShenandoahRegionPartitions; + +// ShenandoahSerialAllocator performs all allocations serially under the heap lock. +// It selects regions from the partitioned free set: +// - Mutator allocations: from Mutator partition with left/right bias alternation +// - Collector allocations: from Collector/OldCollector partition with affiliation preference +// - Overflow: collector may steal empty regions from Mutator partition +// - Humongous: delegated to ShenandoahFreeSet::allocate_contiguous() +class ShenandoahSerialAllocator : public ShenandoahAllocator { +private: + ShenandoahHeap* const _heap; + + // Allocation direction alternates to avoid repeatedly skipping the same uncollected regions. + ssize_t _alloc_bias_weight; + static const ssize_t INITIAL_ALLOC_BIAS_WEIGHT = 256; + + // Attempt allocation within a single region. Handles LAB sizing, PLAB card-alignment, + // affiliation checks, and updates partition accounting via ShenandoahFreeSet. + // Retires the region if remaining capacity falls below PLAB::min_size(). + HeapWord* try_allocate_in(ShenandoahHeapRegion* r, ShenandoahAllocRequest& req, bool& in_new_region); + + HeapWord* allocate_single(ShenandoahAllocRequest& req, bool& in_new_region); + HeapWord* allocate_for_mutator(ShenandoahAllocRequest& req, bool& in_new_region); + HeapWord* allocate_for_collector(ShenandoahAllocRequest& req, bool& in_new_region); + + // Steal an empty region from Mutator partition for collector use. + HeapWord* try_allocate_from_mutator(ShenandoahAllocRequest& req, bool& in_new_region); + + // Re-evaluate left-to-right vs right-to-left bias for mutator allocations. + void update_allocation_bias(); + + // Scan regions using the given iterator, attempt allocation in each. + template + HeapWord* allocate_from_regions(Iter& iterator, ShenandoahAllocRequest& req, bool& in_new_region); + + // For collector: prefer regions matching the requested affiliation, fall back to FREE regions. + template + HeapWord* allocate_with_affiliation(Iter& iterator, + ShenandoahAffiliation affiliation, + ShenandoahAllocRequest& req, + bool& in_new_region); + +public: + ShenandoahSerialAllocator(ShenandoahFreeSet* free_set); + + HeapWord* allocate(ShenandoahAllocRequest& req, bool& in_new_region) override; +}; + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHSERIALALLOCATOR_HPP