From 3e4ca84d664ec8a59e5e87f17fca10574f61005f Mon Sep 17 00:00:00 2001 From: Xiaolong Peng Date: Thu, 28 May 2026 14:52:35 -0700 Subject: [PATCH] Shenandoah: Introduce per-partition allocator instances with retained region optimization --- .../gc/shenandoah/shenandoahAllocator.hpp | 3 + .../share/gc/shenandoah/shenandoahFreeSet.cpp | 3 + .../share/gc/shenandoah/shenandoahFreeSet.hpp | 1 + .../shenandoahPartitionAllocator.cpp | 378 ++++++++++++++++++ .../shenandoahPartitionAllocator.hpp | 88 ++++ .../shenandoah/shenandoahSerialAllocator.cpp | 363 +---------------- .../shenandoah/shenandoahSerialAllocator.hpp | 51 +-- 7 files changed, 497 insertions(+), 390 deletions(-) create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahPartitionAllocator.cpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahPartitionAllocator.hpp diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAllocator.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAllocator.hpp index 4ebf2f8c40e..87b005c2071 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAllocator.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAllocator.hpp @@ -44,6 +44,9 @@ public: // Allocate memory for the given request. Returns nullptr on failure. // Sets in_new_region to true if allocation consumes a previously empty region. virtual HeapWord* allocate(ShenandoahAllocRequest& req, bool& in_new_region) = 0; + + // Called during free-set rebuild to invalidate any cached allocation state. + virtual void clear_retained_regions() = 0; }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHALLOCATOR_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 9fc7234d4d1..4c85ca04fae 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -26,6 +26,7 @@ #include "gc/shared/tlab_globals.hpp" #include "gc/shenandoah/shenandoahAffiliation.hpp" +#include "gc/shenandoah/shenandoahAllocator.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegionSet.hpp" @@ -2604,6 +2605,8 @@ void ShenandoahFreeSet::prepare_to_rebuild(size_t &young_trashed_regions, size_t shenandoah_assert_heaplocked(); assert(rebuild_lock() != nullptr, "sanity"); rebuild_lock()->lock(false); + // Invalidate any retained regions in the allocator before clearing partition state. + _heap->allocator()->clear_retained_regions(); // This resets all state information, removing all regions from all sets. clear(); log_debug(gc, free)("Rebuilding FreeSet"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp index a7e07b47fa7..200e8fef7c8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp @@ -441,6 +441,7 @@ public: class ShenandoahFreeSet : public CHeapObj { friend class ShenandoahAllocator; friend class ShenandoahSerialAllocator; + template friend class ShenandoahPartitionAllocator; using idx_t = ShenandoahSimpleBitMap::idx_t; private: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPartitionAllocator.cpp b/src/hotspot/share/gc/shenandoah/shenandoahPartitionAllocator.cpp new file mode 100644 index 00000000000..f04236f989c --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahPartitionAllocator.cpp @@ -0,0 +1,378 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE THIS COPYRIGHT NOTICE OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "gc/shared/cardTable.hpp" +#include "gc/shared/plab.hpp" +#include "gc/shenandoah/shenandoahAllocRequest.hpp" +#include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" +#include "gc/shenandoah/shenandoahPartitionAllocator.hpp" +#include "logging/log.hpp" + +using idx_t = ShenandoahSimpleBitMap::idx_t; + +class ShenandoahLeftRightIterator { +private: + idx_t _idx; + idx_t _end; + ShenandoahRegionPartitions* _partitions; + ShenandoahFreeSetPartitionId _partition; +public: + explicit ShenandoahLeftRightIterator(ShenandoahRegionPartitions* partitions, + ShenandoahFreeSetPartitionId partition, bool use_empty = false) + : _idx(0), _end(0), _partitions(partitions), _partition(partition) { + _idx = use_empty ? _partitions->leftmost_empty(_partition) : _partitions->leftmost(_partition); + _end = use_empty ? _partitions->rightmost_empty(_partition) : _partitions->rightmost(_partition); + } + + bool has_next() const { + if (_idx <= _end) { + assert(_partitions->in_free_set(_partition, _idx), "Boundaries or find_last_set_bit failed: %zd", _idx); + return true; + } + return false; + } + + idx_t current() const { return _idx; } + + idx_t next() { + _idx = _partitions->find_index_of_next_available_region(_partition, _idx + 1); + return current(); + } +}; + +class ShenandoahRightLeftIterator { +private: + idx_t _idx; + idx_t _end; + ShenandoahRegionPartitions* _partitions; + ShenandoahFreeSetPartitionId _partition; +public: + explicit ShenandoahRightLeftIterator(ShenandoahRegionPartitions* partitions, + ShenandoahFreeSetPartitionId partition, bool use_empty = false) + : _idx(0), _end(0), _partitions(partitions), _partition(partition) { + _idx = use_empty ? _partitions->rightmost_empty(_partition) : _partitions->rightmost(_partition); + _end = use_empty ? _partitions->leftmost_empty(_partition) : _partitions->leftmost(_partition); + } + + bool has_next() const { + if (_idx >= _end) { + assert(_partitions->in_free_set(_partition, _idx), "Boundaries or find_last_set_bit failed: %zd", _idx); + return true; + } + return false; + } + + idx_t current() const { return _idx; } + + idx_t next() { + _idx = _partitions->find_index_of_previous_available_region(_partition, _idx - 1); + return current(); + } +}; + +template +ShenandoahPartitionAllocator::ShenandoahPartitionAllocator(ShenandoahFreeSet* free_set) + : _free_set(free_set), + _heap(ShenandoahHeap::heap()), + _retained_region(nullptr), + _alloc_bias_weight(INITIAL_ALLOC_BIAS_WEIGHT) {} + +template +HeapWord* ShenandoahPartitionAllocator::allocate(ShenandoahAllocRequest& req, bool& in_new_region) { + // Fast path: try the retained region from the previous allocation. + if (_retained_region != nullptr) { + size_t min_size = req.is_lab_alloc() ? req.min_size() : req.size(); + if (_free_set->alloc_capacity(_retained_region) >= min_size * HeapWordSize) { + HeapWord* result = try_allocate_in(_retained_region, req, in_new_region); + if (result != nullptr) { + return result; + } + } + _retained_region = nullptr; + } + + ShenandoahRegionPartitions& partitions = _free_set->_partitions; + + if constexpr (PARTITION == ShenandoahFreeSetPartitionId::Mutator) { + // Mutator allocation: use left/right bias alternation. + update_allocation_bias(); + + if (partitions.is_empty(PARTITION)) { + return nullptr; + } + + if (partitions.alloc_from_left_bias(PARTITION)) { + ShenandoahLeftRightIterator iterator(&partitions, PARTITION); + return allocate_from_regions(iterator, req, in_new_region); + } + + ShenandoahRightLeftIterator iterator(&partitions, PARTITION); + return allocate_from_regions(iterator, req, in_new_region); + + } else { + // Collector/OldCollector: prefer regions matching affiliation, then overflow from Mutator. + HeapWord* result = nullptr; + if (partitions.alloc_from_left_bias(PARTITION)) { + ShenandoahLeftRightIterator iterator(&partitions, PARTITION); + result = allocate_with_affiliation(iterator, req.affiliation(), req, in_new_region); + } else { + ShenandoahRightLeftIterator iterator(&partitions, PARTITION); + result = allocate_with_affiliation(iterator, req.affiliation(), req, in_new_region); + } + + if (result != nullptr) { + return result; + } + + if (!ShenandoahEvacReserveOverflow) { + return nullptr; + } + + // Overflow: flip an empty region from Mutator partition, then allocate in it. + if (partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::Mutator) > 0) { + result = try_allocate_from_mutator(req, in_new_region); + } + return result; + } +} + +// Alternating allocation direction between GC passes improves evacuation performance by +// consuming partially-used regions before they become uncollectable floating garbage. +template +void ShenandoahPartitionAllocator::update_allocation_bias() { + if (_alloc_bias_weight-- <= 0) { + ShenandoahRegionPartitions& partitions = _free_set->_partitions; + idx_t non_empty_on_left = (partitions.leftmost_empty(PARTITION) - partitions.leftmost(PARTITION)); + idx_t non_empty_on_right = (partitions.rightmost(PARTITION) - partitions.rightmost_empty(PARTITION)); + partitions.set_bias_from_left_to_right(PARTITION, (non_empty_on_right < non_empty_on_left)); + _alloc_bias_weight = INITIAL_ALLOC_BIAS_WEIGHT; + } +} + +template +template +HeapWord* ShenandoahPartitionAllocator::allocate_from_regions(Iter& iterator, ShenandoahAllocRequest& req, bool& in_new_region) { + for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) { + ShenandoahHeapRegion* r = _heap->get_region(idx); + size_t min_size = req.is_lab_alloc() ? req.min_size() : req.size(); + if (_free_set->alloc_capacity(r) >= min_size * HeapWordSize) { + HeapWord* result = try_allocate_in(r, req, in_new_region); + if (result != nullptr) { + return result; + } + } + } + return nullptr; +} + +template +template +HeapWord* ShenandoahPartitionAllocator::allocate_with_affiliation(Iter& iterator, + ShenandoahAffiliation affiliation, + ShenandoahAllocRequest& req, + bool& in_new_region) { + assert(affiliation != ShenandoahAffiliation::FREE, "Must not"); + ShenandoahHeapRegion* free_region = nullptr; + for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) { + ShenandoahHeapRegion* r = _heap->get_region(idx); + if (r->affiliation() == affiliation) { + HeapWord* result = try_allocate_in(r, req, in_new_region); + if (result != nullptr) { + return result; + } + } else if (free_region == nullptr && r->affiliation() == FREE) { + free_region = r; + } + } + if (free_region != nullptr) { + HeapWord* result = try_allocate_in(free_region, req, in_new_region); + assert(result != nullptr, "Allocate in free region in the partition always succeed."); + return result; + } + log_debug(gc, free)("Could not allocate collector region with affiliation: %s for request " PTR_FORMAT, + shenandoah_affiliation_name(affiliation), p2i(&req)); + return nullptr; +} + +// Flip an empty region from Mutator to this collector partition, then allocate in it. +// Searches from right to left to keep longer-lived collector regions at high addresses. +template +HeapWord* ShenandoahPartitionAllocator::try_allocate_from_mutator(ShenandoahAllocRequest& req, bool& in_new_region) { + ShenandoahRegionPartitions& partitions = _free_set->_partitions; + ShenandoahRightLeftIterator iterator(&partitions, ShenandoahFreeSetPartitionId::Mutator, true); + for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) { + ShenandoahHeapRegion* r = _heap->get_region(idx); + if (_free_set->can_allocate_from(r)) { + if (req.is_old()) { + if (!_free_set->flip_to_old_gc(r)) { + continue; + } + } else { + _free_set->flip_to_gc(r); + } + log_debug(gc, free)("Flipped region %zu to gc for request: " PTR_FORMAT, idx, p2i(&req)); + return try_allocate_in(r, req, in_new_region); + } + } + return nullptr; +} + +// Allocate within region r. Handles region recycling, LAB sizing, PLAB alignment, +// partition accounting, and region retirement. +template +HeapWord* ShenandoahPartitionAllocator::try_allocate_in(ShenandoahHeapRegion* r, ShenandoahAllocRequest& req, bool& in_new_region) { + assert(_free_set->has_alloc_capacity(r), "Performance: should avoid full regions on this path: %zu", r->index()); + + ShenandoahHeap* heap = _heap; + // Trash regions cannot be used while weak roots processing needs accurate marking info. + if (heap->is_concurrent_weak_root_in_progress() && r->is_trash()) { + return nullptr; + } + + HeapWord* result = nullptr; + r->try_recycle_under_lock(); + in_new_region = r->is_empty(); + if (in_new_region) { + log_debug(gc, free)("Using new region (%zu) for %s (" PTR_FORMAT ").", + r->index(), req.type_string(), p2i(&req)); + assert(!r->is_affiliated(), "New region %zu should be unaffiliated", r->index()); + r->set_affiliation(req.affiliation()); + if (r->is_old()) { + r->end_preemptible_coalesce_and_fill(); + } +#ifdef ASSERT + ShenandoahMarkingContext* const ctx = heap->marking_context(); + assert(ctx->top_at_mark_start(r) == r->bottom(), "Newly established allocation region starts with TAMS equal to bottom"); + assert(ctx->is_bitmap_range_within_region_clear(ctx->top_bitmap(r), r->end()), "Bitmap above top_bitmap() must be clear"); +#endif + } else { + assert(r->is_affiliated(), "Region %zu that is not new should be affiliated", r->index()); + if (r->affiliation() != req.affiliation()) { + assert(heap->mode()->is_generational(), "Request for %s from %s region should only happen in generational mode.", + req.affiliation_name(), r->affiliation_name()); + return nullptr; + } + } + + // Perform the actual allocation: LABs may be shrunk to fit, PLABs must be card-aligned. + if (req.is_lab_alloc()) { + size_t adjusted_size = req.size(); + size_t free = r->free(); + if (req.is_old()) { + assert(heap->mode()->is_generational(), "PLABs are only for generational mode"); + assert(_free_set->_partitions.in_free_set(ShenandoahFreeSetPartitionId::OldCollector, r->index()), + "PLABS must be allocated in old_collector_free regions"); + + size_t usable_free = _free_set->get_usable_free_words(free); + if (adjusted_size > usable_free) { + adjusted_size = usable_free; + } + adjusted_size = align_down(adjusted_size, CardTable::card_size_in_words()); + if (adjusted_size >= req.min_size()) { + result = _free_set->allocate_aligned_plab(adjusted_size, req, r); + assert(result != nullptr, "allocate must succeed"); + req.set_actual_size(adjusted_size); + } else { + log_trace(gc, free)("Failed to shrink PLAB request (%zu) in region %zu to %zu" + " because min_size() is %zu", req.size(), r->index(), adjusted_size, req.min_size()); + } + } else { + free = align_down(free >> LogHeapWordSize, MinObjAlignment); + if (adjusted_size > free) { + adjusted_size = free; + } + if (adjusted_size >= req.min_size()) { + result = r->allocate(adjusted_size, req); + assert(result != nullptr, "Allocation must succeed: free %zu, actual %zu", free, adjusted_size); + req.set_actual_size(adjusted_size); + } else { + log_trace(gc, free)("Failed to shrink TLAB or GCLAB request (%zu) in region %zu to %zu" + " because min_size() is %zu", req.size(), r->index(), adjusted_size, req.min_size()); + } + } + } else { + size_t size = req.size(); + result = r->allocate(size, req); + if (result != nullptr) { + req.set_actual_size(size); + } + } + + // Update partition used bytes on success. + if (result != nullptr) { + if constexpr (PARTITION == ShenandoahFreeSetPartitionId::Mutator) { + assert(req.is_young(), "Mutator allocations always come from young generation."); + _free_set->_partitions.increase_used(PARTITION, req.actual_size() * HeapWordSize); + } else { + assert(req.is_gc_alloc(), "Should be gc_alloc since req wasn't mutator alloc"); + // GC allocations set update_watermark so relocated objects aren't re-updated during update-refs. + r->set_update_watermark(r->top()); + _free_set->_partitions.increase_used(PARTITION, (req.actual_size() + req.waste()) * HeapWordSize); + } + } + + DEBUG_ONLY(bool boundary_changed = false;) + if ((result != nullptr) && in_new_region) { + _free_set->_partitions.one_region_is_no_longer_empty(PARTITION); + DEBUG_ONLY(boundary_changed = true;) + } + + // Retire the region if remaining capacity is too small for any future PLAB. + if (_free_set->alloc_capacity(r) < PLAB::min_size() * HeapWordSize) { + size_t idx = r->index(); + size_t waste_bytes = _free_set->_partitions.retire_from_partition(PARTITION, idx, r->used()); + DEBUG_ONLY(boundary_changed = true;) + if constexpr (PARTITION == ShenandoahFreeSetPartitionId::Mutator) { + if (waste_bytes > 0) { + req.set_waste(waste_bytes / HeapWordSize); + } + } + if (_retained_region == r) { + _retained_region = nullptr; + } + } else if (result != nullptr) { + // Region still has usable capacity — retain for next allocation. + _retained_region = r; + } + + // Recompute generation used/affiliated totals. + _free_set->notify_allocation(PARTITION, in_new_region); + +#ifdef ASSERT + if (boundary_changed) { + _free_set->_partitions.assert_bounds(); + } else { + _free_set->_partitions.assert_bounds_sanity(); + } +#endif + return result; +} + +// Explicit template instantiations for all partitions. +template class ShenandoahPartitionAllocator; +template class ShenandoahPartitionAllocator; +template class ShenandoahPartitionAllocator; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPartitionAllocator.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPartitionAllocator.hpp new file mode 100644 index 00000000000..f4fb8778150 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahPartitionAllocator.hpp @@ -0,0 +1,88 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE THIS COPYRIGHT NOTICE OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHPARTITIONALLOCATOR_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHPARTITIONALLOCATOR_HPP + +#include "gc/shenandoah/shenandoahAffiliation.hpp" +#include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "memory/allocation.hpp" + +class ShenandoahAllocRequest; +class ShenandoahHeap; +class ShenandoahHeapRegion; +class ShenandoahSerialAllocator; + +// ShenandoahPartitionAllocator handles allocation within a single free-set partition. +// Templated on partition ID so that partition-specific behavior (bias for Mutator, +// affiliation preference for Collector/OldCollector) is resolved at compile time. +template +class ShenandoahPartitionAllocator : public CHeapObj { + friend class ShenandoahSerialAllocator; + +private: + ShenandoahFreeSet* const _free_set; + ShenandoahHeap* const _heap; + + // Last region that had remaining capacity after allocation. Checked first on next request + // to avoid scanning the partition bitmap. Cleared on free-set rebuild. + ShenandoahHeapRegion* _retained_region; + + // Allocation direction alternates to pack allocations tightly. Only used for Mutator. + ssize_t _alloc_bias_weight; + static const ssize_t INITIAL_ALLOC_BIAS_WEIGHT = 256; + + // Attempt allocation within a single region. Handles LAB sizing, PLAB card-alignment, + // affiliation checks, and updates partition accounting via ShenandoahFreeSet. + // Retires the region if remaining capacity falls below PLAB::min_size(). + HeapWord* try_allocate_in(ShenandoahHeapRegion* r, ShenandoahAllocRequest& req, bool& in_new_region); + + // Re-evaluate left-to-right vs right-to-left bias. Only meaningful for Mutator partition. + void update_allocation_bias(); + + // Scan regions using the given iterator, attempt allocation in each. + template + HeapWord* allocate_from_regions(Iter& iterator, ShenandoahAllocRequest& req, bool& in_new_region); + + // For collector partitions: prefer regions matching the requested affiliation, fall back to FREE. + template + HeapWord* allocate_with_affiliation(Iter& iterator, + ShenandoahAffiliation affiliation, + ShenandoahAllocRequest& req, + bool& in_new_region); + + // Flip an empty region from Mutator partition to this collector partition, then allocate in it. + HeapWord* try_allocate_from_mutator(ShenandoahAllocRequest& req, bool& in_new_region); + +public: + ShenandoahPartitionAllocator(ShenandoahFreeSet* free_set); + + // Allocate from this partition. Returns nullptr if partition cannot satisfy the request. + HeapWord* allocate(ShenandoahAllocRequest& req, bool& in_new_region); + + // Must be called when the free set is rebuilt to invalidate retained regions. + void clear_retained_regions() { _retained_region = nullptr; } +}; + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHPARTITIONALLOCATOR_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.cpp b/src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.cpp index bd1d88e5b3a..5145c2e7d4c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.cpp @@ -22,89 +22,20 @@ * */ -#include "gc/shared/cardTable.hpp" -#include "gc/shared/plab.hpp" #include "gc/shenandoah/shenandoahAllocRequest.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" -#include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" -#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" #include "gc/shenandoah/shenandoahSerialAllocator.hpp" -#include "logging/log.hpp" - -using idx_t = ShenandoahSimpleBitMap::idx_t; - -class ShenandoahLeftRightIterator { -private: - idx_t _idx; - idx_t _end; - ShenandoahRegionPartitions* _partitions; - ShenandoahFreeSetPartitionId _partition; -public: - explicit ShenandoahLeftRightIterator(ShenandoahRegionPartitions* partitions, - ShenandoahFreeSetPartitionId partition, bool use_empty = false) - : _idx(0), _end(0), _partitions(partitions), _partition(partition) { - _idx = use_empty ? _partitions->leftmost_empty(_partition) : _partitions->leftmost(_partition); - _end = use_empty ? _partitions->rightmost_empty(_partition) : _partitions->rightmost(_partition); - } - - bool has_next() const { - if (_idx <= _end) { - assert(_partitions->in_free_set(_partition, _idx), "Boundaries or find_last_set_bit failed: %zd", _idx); - return true; - } - return false; - } - - idx_t current() const { - return _idx; - } - - idx_t next() { - _idx = _partitions->find_index_of_next_available_region(_partition, _idx + 1); - return current(); - } -}; - -class ShenandoahRightLeftIterator { -private: - idx_t _idx; - idx_t _end; - ShenandoahRegionPartitions* _partitions; - ShenandoahFreeSetPartitionId _partition; -public: - explicit ShenandoahRightLeftIterator(ShenandoahRegionPartitions* partitions, - ShenandoahFreeSetPartitionId partition, bool use_empty = false) - : _idx(0), _end(0), _partitions(partitions), _partition(partition) { - _idx = use_empty ? _partitions->rightmost_empty(_partition) : _partitions->rightmost(_partition); - _end = use_empty ? _partitions->leftmost_empty(_partition) : _partitions->leftmost(_partition); - } - - bool has_next() const { - if (_idx >= _end) { - assert(_partitions->in_free_set(_partition, _idx), "Boundaries or find_last_set_bit failed: %zd", _idx); - return true; - } - return false; - } - - idx_t current() const { - return _idx; - } - - idx_t next() { - _idx = _partitions->find_index_of_previous_available_region(_partition, _idx - 1); - return current(); - } -}; ShenandoahSerialAllocator::ShenandoahSerialAllocator(ShenandoahFreeSet* free_set) : ShenandoahAllocator(free_set), - _heap(ShenandoahHeap::heap()), - _alloc_bias_weight(INITIAL_ALLOC_BIAS_WEIGHT) {} + _mutator_alloc(free_set), + _collector_alloc(free_set), + _old_collector_alloc(free_set) {} HeapWord* ShenandoahSerialAllocator::allocate(ShenandoahAllocRequest& req, bool& in_new_region) { shenandoah_assert_heaplocked(); + if (ShenandoahHeapRegion::requires_humongous(req.size())) { switch (req.type()) { case ShenandoahAllocRequest::_alloc_shared: @@ -124,286 +55,20 @@ HeapWord* ShenandoahSerialAllocator::allocate(ShenandoahAllocRequest& req, bool& ShouldNotReachHere(); return nullptr; } - } else { - return allocate_single(req, in_new_region); } -} - -HeapWord* ShenandoahSerialAllocator::allocate_single(ShenandoahAllocRequest& req, bool& in_new_region) { - shenandoah_assert_heaplocked(); + // Dispatch to the appropriate per-partition allocator. if (req.is_mutator_alloc()) { - return allocate_for_mutator(req, in_new_region); - } else { - return allocate_for_collector(req, in_new_region); - } -} - -HeapWord* ShenandoahSerialAllocator::allocate_for_mutator(ShenandoahAllocRequest& req, bool& in_new_region) { - update_allocation_bias(); - - ShenandoahRegionPartitions& partitions = _free_set->_partitions; - if (partitions.is_empty(ShenandoahFreeSetPartitionId::Mutator)) { - return nullptr; - } - - if (partitions.alloc_from_left_bias(ShenandoahFreeSetPartitionId::Mutator)) { - ShenandoahLeftRightIterator iterator(&partitions, ShenandoahFreeSetPartitionId::Mutator); - return allocate_from_regions(iterator, req, in_new_region); - } - - ShenandoahRightLeftIterator iterator(&partitions, ShenandoahFreeSetPartitionId::Mutator); - return allocate_from_regions(iterator, req, in_new_region); -} - -// Alternating allocation direction between GC passes improves evacuation performance by -// consuming partially-used regions before they become uncollectable floating garbage. -// We bias toward the side with fewer non-empty regions to pack allocations tightly. -void ShenandoahSerialAllocator::update_allocation_bias() { - if (_alloc_bias_weight-- <= 0) { - ShenandoahRegionPartitions& partitions = _free_set->_partitions; - - idx_t non_empty_on_left = (partitions.leftmost_empty(ShenandoahFreeSetPartitionId::Mutator) - - partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator)); - idx_t non_empty_on_right = (partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator) - - partitions.rightmost_empty(ShenandoahFreeSetPartitionId::Mutator)); - partitions.set_bias_from_left_to_right(ShenandoahFreeSetPartitionId::Mutator, (non_empty_on_right < non_empty_on_left)); - _alloc_bias_weight = INITIAL_ALLOC_BIAS_WEIGHT; - } -} - -template -HeapWord* ShenandoahSerialAllocator::allocate_from_regions(Iter& iterator, ShenandoahAllocRequest& req, bool& in_new_region) { - for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) { - ShenandoahHeapRegion* r = _heap->get_region(idx); - size_t min_size = req.is_lab_alloc() ? req.min_size() : req.size(); - if (_free_set->alloc_capacity(r) >= min_size * HeapWordSize) { - HeapWord* result = try_allocate_in(r, req, in_new_region); - if (result != nullptr) { - return result; - } - } - } - return nullptr; -} - -// Collector allocation: first try the reserved Collector/OldCollector partition, -// preferring regions with matching affiliation. If that fails and ShenandoahEvacReserveOverflow -// is enabled, steal an empty region from the Mutator partition. -HeapWord* ShenandoahSerialAllocator::allocate_for_collector(ShenandoahAllocRequest& req, bool& in_new_region) { - shenandoah_assert_heaplocked(); - ShenandoahRegionPartitions& partitions = _free_set->_partitions; - ShenandoahFreeSetPartitionId which_partition = req.is_old() ? ShenandoahFreeSetPartitionId::OldCollector - : ShenandoahFreeSetPartitionId::Collector; - HeapWord* result = nullptr; - if (partitions.alloc_from_left_bias(which_partition)) { - ShenandoahLeftRightIterator iterator(&partitions, which_partition); - result = allocate_with_affiliation(iterator, req.affiliation(), req, in_new_region); - } else { - ShenandoahRightLeftIterator iterator(&partitions, which_partition); - result = allocate_with_affiliation(iterator, req.affiliation(), req, in_new_region); - } - - if (result != nullptr) { - return result; - } - - if (!ShenandoahEvacReserveOverflow) { - return nullptr; - } - - // Overflow: steal empty region from mutator partition for collector use. - if (partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::Mutator) > 0) { - result = try_allocate_from_mutator(req, in_new_region); - } - - return result; -} - -template -HeapWord* ShenandoahSerialAllocator::allocate_with_affiliation(Iter& iterator, - ShenandoahAffiliation affiliation, - ShenandoahAllocRequest& req, - bool& in_new_region) { - assert(affiliation != ShenandoahAffiliation::FREE, "Must not"); - ShenandoahHeapRegion* free_region = nullptr; - for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) { - ShenandoahHeapRegion* r = _heap->get_region(idx); - if (r->affiliation() == affiliation) { - HeapWord* result = try_allocate_in(r, req, in_new_region); - if (result != nullptr) { - return result; - } - } else if (free_region == nullptr && r->affiliation() == FREE) { - free_region = r; - } - } - if (free_region != nullptr) { - HeapWord* result = try_allocate_in(free_region, req, in_new_region); - assert(result != nullptr, "Allocate in free region in the partition always succeed."); - return result; - } - log_debug(gc, free)("Could not allocate collector region with affiliation: %s for request " PTR_FORMAT, - shenandoah_affiliation_name(affiliation), p2i(&req)); - return nullptr; -} - -// Flip an empty region from Mutator to Collector/OldCollector partition, then allocate in it. -// Searches from right to left to keep longer-lived collector regions at high addresses. -HeapWord* ShenandoahSerialAllocator::try_allocate_from_mutator(ShenandoahAllocRequest& req, bool& in_new_region) { - ShenandoahRegionPartitions& partitions = _free_set->_partitions; - ShenandoahRightLeftIterator iterator(&partitions, ShenandoahFreeSetPartitionId::Mutator, true); - for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) { - ShenandoahHeapRegion* r = _heap->get_region(idx); - if (_free_set->can_allocate_from(r)) { - if (req.is_old()) { - if (!_free_set->flip_to_old_gc(r)) { - continue; - } - } else { - _free_set->flip_to_gc(r); - } - log_debug(gc, free)("Flipped region %zu to gc for request: " PTR_FORMAT, idx, p2i(&req)); - return try_allocate_in(r, req, in_new_region); - } - } - - return nullptr; -} - -// Allocate within region r for the given request. This handles: -// 1. Region recycling and affiliation setup for new (empty) regions -// 2. LAB sizing (TLAB/GCLAB shrink-to-fit, PLAB card-alignment) -// 3. Partition accounting (used, empty counts, retirement) -HeapWord* ShenandoahSerialAllocator::try_allocate_in(ShenandoahHeapRegion* r, ShenandoahAllocRequest& req, bool& in_new_region) { - assert(_free_set->has_alloc_capacity(r), "Performance: should avoid full regions on this path: %zu", r->index()); - - ShenandoahHeap* heap = ShenandoahHeap::heap(); - // Trash regions cannot be used while weak roots processing needs accurate marking info. - if (heap->is_concurrent_weak_root_in_progress() && r->is_trash()) { - return nullptr; - } - - HeapWord* result = nullptr; - r->try_recycle_under_lock(); - in_new_region = r->is_empty(); - if (in_new_region) { - log_debug(gc, free)("Using new region (%zu) for %s (" PTR_FORMAT ").", - r->index(), req.type_string(), p2i(&req)); - assert(!r->is_affiliated(), "New region %zu should be unaffiliated", r->index()); - r->set_affiliation(req.affiliation()); - if (r->is_old()) { - r->end_preemptible_coalesce_and_fill(); - } -#ifdef ASSERT - ShenandoahMarkingContext* const ctx = heap->marking_context(); - assert(ctx->top_at_mark_start(r) == r->bottom(), "Newly established allocation region starts with TAMS equal to bottom"); - assert(ctx->is_bitmap_range_within_region_clear(ctx->top_bitmap(r), r->end()), "Bitmap above top_bitmap() must be clear"); -#endif - } else { - assert(r->is_affiliated(), "Region %zu that is not new should be affiliated", r->index()); - if (r->affiliation() != req.affiliation()) { - assert(heap->mode()->is_generational(), "Request for %s from %s region should only happen in generational mode.", - req.affiliation_name(), r->affiliation_name()); - return nullptr; - } - } - - // Perform the actual allocation: LABs may be shrunk to fit, PLABs must be card-aligned. - if (req.is_lab_alloc()) { - size_t adjusted_size = req.size(); - size_t free = r->free(); - if (req.is_old()) { - assert(heap->mode()->is_generational(), "PLABs are only for generational mode"); - assert(_free_set->_partitions.in_free_set(ShenandoahFreeSetPartitionId::OldCollector, r->index()), - "PLABS must be allocated in old_collector_free regions"); - - size_t usable_free = _free_set->get_usable_free_words(free); - if (adjusted_size > usable_free) { - adjusted_size = usable_free; - } - adjusted_size = align_down(adjusted_size, CardTable::card_size_in_words()); - if (adjusted_size >= req.min_size()) { - result = _free_set->allocate_aligned_plab(adjusted_size, req, r); - assert(result != nullptr, "allocate must succeed"); - req.set_actual_size(adjusted_size); - } else { - log_trace(gc, free)("Failed to shrink PLAB request (%zu) in region %zu to %zu" - " because min_size() is %zu", req.size(), r->index(), adjusted_size, req.min_size()); - } - } else { - free = align_down(free >> LogHeapWordSize, MinObjAlignment); - if (adjusted_size > free) { - adjusted_size = free; - } - if (adjusted_size >= req.min_size()) { - result = r->allocate(adjusted_size, req); - assert(result != nullptr, "Allocation must succeed: free %zu, actual %zu", free, adjusted_size); - req.set_actual_size(adjusted_size); - } else { - log_trace(gc, free)("Failed to shrink TLAB or GCLAB request (%zu) in region %zu to %zu" - " because min_size() is %zu", req.size(), r->index(), adjusted_size, req.min_size()); - } - } - } else { - size_t size = req.size(); - result = r->allocate(size, req); - if (result != nullptr) { - req.set_actual_size(size); - } - } - - // Update partition used bytes on success. - if (result != nullptr) { - if (req.is_mutator_alloc()) { - assert(req.is_young(), "Mutator allocations always come from young generation."); - _free_set->_partitions.increase_used(ShenandoahFreeSetPartitionId::Mutator, req.actual_size() * HeapWordSize); - } else { - assert(req.is_gc_alloc(), "Should be gc_alloc since req wasn't mutator alloc"); - // GC allocations set update_watermark so relocated objects aren't re-updated during update-refs. - r->set_update_watermark(r->top()); - if (r->is_old()) { - _free_set->_partitions.increase_used(ShenandoahFreeSetPartitionId::OldCollector, (req.actual_size() + req.waste()) * HeapWordSize); - } else { - _free_set->_partitions.increase_used(ShenandoahFreeSetPartitionId::Collector, (req.actual_size() + req.waste()) * HeapWordSize); - } - } - } - - ShenandoahFreeSetPartitionId orig_partition; - if (req.is_mutator_alloc()) { - orig_partition = ShenandoahFreeSetPartitionId::Mutator; + return _mutator_alloc.allocate(req, in_new_region); } else if (req.is_old()) { - orig_partition = ShenandoahFreeSetPartitionId::OldCollector; + return _old_collector_alloc.allocate(req, in_new_region); } else { - orig_partition = ShenandoahFreeSetPartitionId::Collector; + return _collector_alloc.allocate(req, in_new_region); } - - DEBUG_ONLY(bool boundary_changed = false;) - if ((result != nullptr) && in_new_region) { - _free_set->_partitions.one_region_is_no_longer_empty(orig_partition); - DEBUG_ONLY(boundary_changed = true;) - } - - // Retire the region if remaining capacity is too small for any future PLAB. - if (_free_set->alloc_capacity(r) < PLAB::min_size() * HeapWordSize) { - size_t idx = r->index(); - size_t waste_bytes = _free_set->_partitions.retire_from_partition(orig_partition, idx, r->used()); - DEBUG_ONLY(boundary_changed = true;) - if (req.is_mutator_alloc() && (waste_bytes > 0)) { - req.set_waste(waste_bytes / HeapWordSize); - } - } - - // Recompute generation used/affiliated totals. - _free_set->notify_allocation(orig_partition, in_new_region); - -#ifdef ASSERT - if (boundary_changed) { - _free_set->_partitions.assert_bounds(); - } else { - _free_set->_partitions.assert_bounds_sanity(); - } -#endif - return result; +} + +void ShenandoahSerialAllocator::clear_retained_regions() { + _mutator_alloc.clear_retained_regions(); + _collector_alloc.clear_retained_regions(); + _old_collector_alloc.clear_retained_regions(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.hpp b/src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.hpp index e911d637a04..cfea41e2742 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSerialAllocator.hpp @@ -25,58 +25,27 @@ #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHSERIALALLOCATOR_HPP #define SHARE_GC_SHENANDOAH_SHENANDOAHSERIALALLOCATOR_HPP -#include "gc/shenandoah/shenandoahAffiliation.hpp" #include "gc/shenandoah/shenandoahAllocator.hpp" - -class ShenandoahFreeSet; -class ShenandoahHeap; -class ShenandoahHeapRegion; -class ShenandoahRegionPartitions; +#include "gc/shenandoah/shenandoahPartitionAllocator.hpp" // ShenandoahSerialAllocator performs all allocations serially under the heap lock. -// It selects regions from the partitioned free set: -// - Mutator allocations: from Mutator partition with left/right bias alternation -// - Collector allocations: from Collector/OldCollector partition with affiliation preference -// - Overflow: collector may steal empty regions from Mutator partition +// It dispatches requests to per-partition allocators based on request type: +// - Mutator allocations: routed to the Mutator partition allocator +// - Collector allocations: routed to Collector or OldCollector partition allocator // - Humongous: delegated to ShenandoahFreeSet::allocate_contiguous() class ShenandoahSerialAllocator : public ShenandoahAllocator { private: - ShenandoahHeap* const _heap; - - // Allocation direction alternates to avoid repeatedly skipping the same uncollected regions. - ssize_t _alloc_bias_weight; - static const ssize_t INITIAL_ALLOC_BIAS_WEIGHT = 256; - - // Attempt allocation within a single region. Handles LAB sizing, PLAB card-alignment, - // affiliation checks, and updates partition accounting via ShenandoahFreeSet. - // Retires the region if remaining capacity falls below PLAB::min_size(). - HeapWord* try_allocate_in(ShenandoahHeapRegion* r, ShenandoahAllocRequest& req, bool& in_new_region); - - HeapWord* allocate_single(ShenandoahAllocRequest& req, bool& in_new_region); - HeapWord* allocate_for_mutator(ShenandoahAllocRequest& req, bool& in_new_region); - HeapWord* allocate_for_collector(ShenandoahAllocRequest& req, bool& in_new_region); - - // Steal an empty region from Mutator partition for collector use. - HeapWord* try_allocate_from_mutator(ShenandoahAllocRequest& req, bool& in_new_region); - - // Re-evaluate left-to-right vs right-to-left bias for mutator allocations. - void update_allocation_bias(); - - // Scan regions using the given iterator, attempt allocation in each. - template - HeapWord* allocate_from_regions(Iter& iterator, ShenandoahAllocRequest& req, bool& in_new_region); - - // For collector: prefer regions matching the requested affiliation, fall back to FREE regions. - template - HeapWord* allocate_with_affiliation(Iter& iterator, - ShenandoahAffiliation affiliation, - ShenandoahAllocRequest& req, - bool& in_new_region); + ShenandoahPartitionAllocator _mutator_alloc; + ShenandoahPartitionAllocator _collector_alloc; + ShenandoahPartitionAllocator _old_collector_alloc; public: ShenandoahSerialAllocator(ShenandoahFreeSet* free_set); HeapWord* allocate(ShenandoahAllocRequest& req, bool& in_new_region) override; + + // Called during free-set rebuild to invalidate retained regions. + void clear_retained_regions() override; }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHSERIALALLOCATOR_HPP