From 9a0fde4a4992bc2a67b66bc86aeceaebe0b77cf4 Mon Sep 17 00:00:00 2001 From: Thomas Schatzl Date: Mon, 23 Mar 2026 11:04:12 +0000 Subject: [PATCH] 8367765: G1: Merge G1UpdateRegionLivenessAndSelectForRebuild task and collection set choosing Reviewed-by: ayang, iwalulya --- .../share/gc/g1/g1CollectionSetCandidates.cpp | 10 +- .../share/gc/g1/g1CollectionSetCandidates.hpp | 5 +- .../share/gc/g1/g1CollectionSetChooser.cpp | 296 ------------------ .../share/gc/g1/g1CollectionSetChooser.hpp | 55 ---- src/hotspot/share/gc/g1/g1ConcurrentMark.cpp | 8 +- .../gc/g1/g1ConcurrentMarkRemarkTasks.cpp | 107 ++++++- .../gc/g1/g1ConcurrentMarkRemarkTasks.hpp | 13 +- src/hotspot/share/gc/g1/g1Policy.cpp | 1 - .../share/gc/g1/g1RemSetTrackingPolicy.cpp | 11 +- 9 files changed, 122 insertions(+), 384 deletions(-) delete mode 100644 src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp delete mode 100644 src/hotspot/share/gc/g1/g1CollectionSetChooser.hpp diff --git a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp index d71108d4d0e..debb0bb54b7 100644 --- a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp +++ b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,7 +23,6 @@ */ #include "gc/g1/g1CollectionSetCandidates.inline.hpp" -#include "gc/g1/g1CollectionSetChooser.hpp" #include "gc/g1/g1HeapRegion.inline.hpp" #include "utilities/growableArray.hpp" @@ -250,8 +249,9 @@ void G1CollectionSetCandidates::sort_marking_by_efficiency() { _from_marking_groups.verify(); } -void G1CollectionSetCandidates::set_candidates_from_marking(G1HeapRegion** candidates, - uint num_candidates) { +void G1CollectionSetCandidates::set_candidates_from_marking(GrowableArrayCHeap* candidates) { + uint num_candidates = candidates->length(); + if (num_candidates == 0) { log_debug(gc, ergo, cset) ("No regions selected from marking."); return; @@ -273,7 +273,7 @@ void G1CollectionSetCandidates::set_candidates_from_marking(G1HeapRegion** candi current = new G1CSetCandidateGroup(); for (uint i = 0; i < num_candidates; i++) { - G1HeapRegion* r = candidates[i]; + G1HeapRegion* r = candidates->at(i); assert(!contains(r), "must not contain region %u", r->hrm_index()); _contains_map[r->hrm_index()] = CandidateOrigin::Marking; diff --git a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.hpp b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.hpp index 7e882de7e5a..8a2235cf89c 100644 --- a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.hpp +++ b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -245,8 +245,7 @@ public: // Merge collection set candidates from marking into the current marking candidates // (which needs to be empty). - void set_candidates_from_marking(G1HeapRegion** candidates, - uint num_candidates); + void set_candidates_from_marking(GrowableArrayCHeap* selected); // The most recent length of the list that had been merged last via // set_candidates_from_marking(). Used for calculating minimum collection set // regions. diff --git a/src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp b/src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp deleted file mode 100644 index e7bab32129e..00000000000 --- a/src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "gc/g1/g1CollectedHeap.inline.hpp" -#include "gc/g1/g1CollectionSetCandidates.hpp" -#include "gc/g1/g1CollectionSetChooser.hpp" -#include "gc/g1/g1HeapRegionRemSet.inline.hpp" -#include "gc/shared/space.hpp" -#include "runtime/atomic.hpp" -#include "utilities/quickSort.hpp" - -// Determine collection set candidates (from marking): For all regions determine -// whether they should be a collection set candidate. Calculate their efficiency, -// sort, and put them into the collection set candidates. -// -// Threads calculate the GC efficiency of the regions they get to process, and -// put them into some work area without sorting. At the end that array is sorted and -// moved to the destination. -class G1BuildCandidateRegionsTask : public WorkerTask { - // Work area for building the set of collection set candidates. Contains references - // to heap regions with their GC efficiencies calculated. To reduce contention - // on claiming array elements, worker threads claim parts of this array in chunks; - // Array elements may be null as threads might not get enough regions to fill - // up their chunks completely. - // Final sorting will remove them. - class G1BuildCandidateArray : public StackObj { - uint const _max_size; - uint const _chunk_size; - - G1HeapRegion** _data; - - Atomic _cur_claim_idx; - - static int compare_region_gc_efficiency(G1HeapRegion** rr1, G1HeapRegion** rr2) { - G1HeapRegion* r1 = *rr1; - G1HeapRegion* r2 = *rr2; - // Make sure that null entries are moved to the end. - if (r1 == nullptr) { - if (r2 == nullptr) { - return 0; - } else { - return 1; - } - } else if (r2 == nullptr) { - return -1; - } - - G1Policy* p = G1CollectedHeap::heap()->policy(); - double gc_efficiency1 = p->predict_gc_efficiency(r1); - double gc_efficiency2 = p->predict_gc_efficiency(r2); - - if (gc_efficiency1 > gc_efficiency2) { - return -1; - } else if (gc_efficiency1 < gc_efficiency2) { - return 1; - } else { - return 0; - } - } - - // Calculates the maximum array size that will be used. - static uint required_array_size(uint num_regions, uint chunk_size, uint num_workers) { - uint const max_waste = num_workers * chunk_size; - // The array should be aligned with respect to chunk_size. - uint const aligned_num_regions = ((num_regions + chunk_size - 1) / chunk_size) * chunk_size; - - return aligned_num_regions + max_waste; - } - - public: - G1BuildCandidateArray(uint max_num_regions, uint chunk_size, uint num_workers) : - _max_size(required_array_size(max_num_regions, chunk_size, num_workers)), - _chunk_size(chunk_size), - _data(NEW_C_HEAP_ARRAY(G1HeapRegion*, _max_size, mtGC)), - _cur_claim_idx(0) { - for (uint i = 0; i < _max_size; i++) { - _data[i] = nullptr; - } - } - - ~G1BuildCandidateArray() { - FREE_C_HEAP_ARRAY(G1HeapRegion*, _data); - } - - // Claim a new chunk, returning its bounds [from, to[. - void claim_chunk(uint& from, uint& to) { - uint result = _cur_claim_idx.add_then_fetch(_chunk_size); - assert(_max_size > result - 1, - "Array too small, is %u should be %u with chunk size %u.", - _max_size, result, _chunk_size); - from = result - _chunk_size; - to = result; - } - - // Set element in array. - void set(uint idx, G1HeapRegion* hr) { - assert(idx < _max_size, "Index %u out of bounds %u", idx, _max_size); - assert(_data[idx] == nullptr, "Value must not have been set."); - _data[idx] = hr; - } - - void sort_by_gc_efficiency() { - uint length = _cur_claim_idx.load_relaxed(); - if (length == 0) { - return; - } - for (uint i = length; i < _max_size; i++) { - assert(_data[i] == nullptr, "must be"); - } - qsort(_data, length, sizeof(_data[0]), (_sort_Fn)compare_region_gc_efficiency); - for (uint i = length; i < _max_size; i++) { - assert(_data[i] == nullptr, "must be"); - } - } - - G1HeapRegion** array() const { return _data; } - }; - - // Per-region closure. In addition to determining whether a region should be - // added to the candidates, and calculating those regions' gc efficiencies, also - // gather additional statistics. - class G1BuildCandidateRegionsClosure : public G1HeapRegionClosure { - G1BuildCandidateArray* _array; - - uint _cur_chunk_idx; - uint _cur_chunk_end; - - uint _regions_added; - - void add_region(G1HeapRegion* hr) { - if (_cur_chunk_idx == _cur_chunk_end) { - _array->claim_chunk(_cur_chunk_idx, _cur_chunk_end); - } - assert(_cur_chunk_idx < _cur_chunk_end, "Must be"); - - _array->set(_cur_chunk_idx, hr); - _cur_chunk_idx++; - - _regions_added++; - } - - public: - G1BuildCandidateRegionsClosure(G1BuildCandidateArray* array) : - _array(array), - _cur_chunk_idx(0), - _cur_chunk_end(0), - _regions_added(0) { } - - bool do_heap_region(G1HeapRegion* r) { - // Candidates from marking are always old; also keep regions that are already - // collection set candidates (some retained regions) in that list. - if (!r->is_old() || r->is_collection_set_candidate()) { - // Keep remembered sets and everything for these regions. - return false; - } - - // Can not add a region without a remembered set to the candidates. - if (!r->rem_set()->is_tracked()) { - return false; - } - - // Skip any region that is currently used as an old GC alloc region. We should - // not consider those for collection before we fill them up as the effective - // gain from them is small. I.e. we only actually reclaim from the filled part, - // as the remainder is still eligible for allocation. These objects are also - // likely to have already survived a few collections, so they might be longer - // lived anyway. - // Otherwise the Old region must satisfy the liveness condition. - bool should_add = !G1CollectedHeap::heap()->is_old_gc_alloc_region(r) && - G1CollectionSetChooser::region_occupancy_low_enough_for_evac(r->live_bytes()); - if (should_add) { - add_region(r); - } else { - r->rem_set()->clear(true /* only_cardset */); - } - return false; - } - - uint regions_added() const { return _regions_added; } - }; - - G1CollectedHeap* _g1h; - G1HeapRegionClaimer _hrclaimer; - - Atomic _num_regions_added; - - G1BuildCandidateArray _result; - - void update_totals(uint num_regions) { - if (num_regions > 0) { - _num_regions_added.add_then_fetch(num_regions); - } - } - - // Early prune (remove) regions meeting the G1HeapWastePercent criteria. That - // is, either until only the minimum amount of old collection set regions are - // available (for forward progress in evacuation) or the waste accumulated by the - // removed regions is above the maximum allowed waste. - // Updates number of candidates and reclaimable bytes given. - void prune(G1HeapRegion** data) { - G1Policy* p = G1CollectedHeap::heap()->policy(); - - uint num_candidates = _num_regions_added.load_relaxed(); - - uint min_old_cset_length = p->calc_min_old_cset_length(num_candidates); - uint num_pruned = 0; - size_t wasted_bytes = 0; - - if (min_old_cset_length >= num_candidates) { - // We take all of the candidate regions to provide some forward progress. - return; - } - - size_t allowed_waste = p->allowed_waste_in_collection_set(); - uint max_to_prune = num_candidates - min_old_cset_length; - - while (true) { - G1HeapRegion* r = data[num_candidates - num_pruned - 1]; - size_t const reclaimable = r->reclaimable_bytes(); - if (num_pruned >= max_to_prune || - wasted_bytes + reclaimable > allowed_waste) { - break; - } - r->rem_set()->clear(true /* cardset_only */); - - wasted_bytes += reclaimable; - num_pruned++; - } - - log_debug(gc, ergo, cset)("Pruned %u regions out of %u, leaving %zu bytes waste (allowed %zu)", - num_pruned, - num_candidates, - wasted_bytes, - allowed_waste); - - _num_regions_added.sub_then_fetch(num_pruned, memory_order_relaxed); - } - -public: - G1BuildCandidateRegionsTask(uint max_num_regions, uint chunk_size, uint num_workers) : - WorkerTask("G1 Build Candidate Regions"), - _g1h(G1CollectedHeap::heap()), - _hrclaimer(num_workers), - _num_regions_added(0), - _result(max_num_regions, chunk_size, num_workers) { } - - void work(uint worker_id) { - G1BuildCandidateRegionsClosure cl(&_result); - _g1h->heap_region_par_iterate_from_worker_offset(&cl, &_hrclaimer, worker_id); - update_totals(cl.regions_added()); - } - - void sort_and_prune_into(G1CollectionSetCandidates* candidates) { - _result.sort_by_gc_efficiency(); - prune(_result.array()); - candidates->set_candidates_from_marking(_result.array(), - _num_regions_added.load_relaxed()); - } -}; - -uint G1CollectionSetChooser::calculate_work_chunk_size(uint num_workers, uint num_regions) { - assert(num_workers > 0, "Active gc workers should be greater than 0"); - return MAX2(num_regions / num_workers, 1U); -} - -void G1CollectionSetChooser::build(WorkerThreads* workers, uint max_num_regions, G1CollectionSetCandidates* candidates) { - uint num_workers = workers->active_workers(); - uint chunk_size = calculate_work_chunk_size(num_workers, max_num_regions); - - G1BuildCandidateRegionsTask cl(max_num_regions, chunk_size, num_workers); - workers->run_task(&cl, num_workers); - - cl.sort_and_prune_into(candidates); - candidates->verify(); -} diff --git a/src/hotspot/share/gc/g1/g1CollectionSetChooser.hpp b/src/hotspot/share/gc/g1/g1CollectionSetChooser.hpp deleted file mode 100644 index 4db8ed23c49..00000000000 --- a/src/hotspot/share/gc/g1/g1CollectionSetChooser.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_GC_G1_G1COLLECTIONSETCHOOSER_HPP -#define SHARE_GC_G1_G1COLLECTIONSETCHOOSER_HPP - -#include "gc/g1/g1HeapRegion.hpp" -#include "gc/shared/gc_globals.hpp" -#include "memory/allStatic.hpp" -#include "runtime/globals.hpp" - -class G1CollectionSetCandidates; -class WorkerThreads; - -// Helper class to calculate collection set candidates, and containing some related -// methods. -class G1CollectionSetChooser : public AllStatic { - static uint calculate_work_chunk_size(uint num_workers, uint num_regions); - - static size_t mixed_gc_live_threshold_bytes() { - return G1HeapRegion::GrainBytes * (size_t)G1MixedGCLiveThresholdPercent / 100; - } - -public: - static bool region_occupancy_low_enough_for_evac(size_t live_bytes) { - return live_bytes < mixed_gc_live_threshold_bytes(); - } - - // Build and return set of collection set candidates sorted by decreasing gc - // efficiency. - static void build(WorkerThreads* workers, uint max_num_regions, G1CollectionSetCandidates* candidates); -}; - -#endif // SHARE_GC_G1_G1COLLECTIONSETCHOOSER_HPP diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp index ab28feb520f..8fd355615d0 100644 --- a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp +++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp @@ -30,7 +30,6 @@ #include "gc/g1/g1CardSetMemory.hpp" #include "gc/g1/g1CardTableClaimTable.inline.hpp" #include "gc/g1/g1CollectedHeap.inline.hpp" -#include "gc/g1/g1CollectionSetChooser.hpp" #include "gc/g1/g1CollectorState.hpp" #include "gc/g1/g1ConcurrentMark.inline.hpp" #include "gc/g1/g1ConcurrentMarkRemarkTasks.hpp" @@ -1328,14 +1327,13 @@ void G1ConcurrentMark::remark() { _g1h->workers()->run_task(&cl, num_workers); log_debug(gc, remset, tracking)("Remembered Set Tracking update regions total %u, selected %u", - _g1h->num_committed_regions(), cl.total_selected_for_rebuild()); + _g1h->num_committed_regions(), cl.total_selected_for_rebuild()); _needs_remembered_set_rebuild = (cl.total_selected_for_rebuild() > 0); if (_needs_remembered_set_rebuild) { - // Prune rebuild candidates based on G1HeapWastePercent. - // Improves rebuild time in addition to remembered set memory usage. - G1CollectionSetChooser::build(_g1h->workers(), _g1h->num_committed_regions(), _g1h->policy()->candidates()); + GrowableArrayCHeap* selected = cl.sort_and_prune_old_selected(); + _g1h->policy()->candidates()->set_candidates_from_marking(selected); } } diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.cpp index 4eb11f6d8f6..3eda7200e25 100644 --- a/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.cpp +++ b/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.cpp @@ -31,28 +31,32 @@ #include "gc/g1/g1RemSetTrackingPolicy.hpp" #include "logging/log.hpp" #include "runtime/mutexLocker.hpp" +#include "utilities/growableArray.hpp" struct G1UpdateRegionLivenessAndSelectForRebuildTask::G1OnRegionClosure : public G1HeapRegionClosure { G1CollectedHeap* _g1h; G1ConcurrentMark* _cm; - // The number of regions actually selected for rebuild. - uint _num_selected_for_rebuild; size_t _freed_bytes; uint _num_old_regions_removed; uint _num_humongous_regions_removed; - G1FreeRegionList* _local_cleanup_list; + + GrowableArrayCHeap _old_selected_for_rebuild; + uint _num_humongous_selected_for_rebuild; + + G1FreeRegionList* _cleanup_list; G1OnRegionClosure(G1CollectedHeap* g1h, G1ConcurrentMark* cm, G1FreeRegionList* local_cleanup_list) : _g1h(g1h), _cm(cm), - _num_selected_for_rebuild(0), _freed_bytes(0), _num_old_regions_removed(0), _num_humongous_regions_removed(0), - _local_cleanup_list(local_cleanup_list) {} + _old_selected_for_rebuild(16), + _num_humongous_selected_for_rebuild(0), + _cleanup_list(local_cleanup_list) {} void reclaim_empty_region_common(G1HeapRegion* hr) { assert(!hr->has_pinned_objects(), "precondition"); @@ -74,7 +78,7 @@ struct G1UpdateRegionLivenessAndSelectForRebuildTask::G1OnRegionClosure : public _num_humongous_regions_removed++; reclaim_empty_region_common(hr); - _g1h->free_humongous_region(hr, _local_cleanup_list); + _g1h->free_humongous_region(hr, _cleanup_list); }; _g1h->humongous_obj_regions_iterate(hr, on_humongous_region); @@ -85,7 +89,7 @@ struct G1UpdateRegionLivenessAndSelectForRebuildTask::G1OnRegionClosure : public _num_old_regions_removed++; reclaim_empty_region_common(hr); - _g1h->free_region(hr, _local_cleanup_list); + _g1h->free_region(hr, _cleanup_list); } bool do_heap_region(G1HeapRegion* hr) override { @@ -98,13 +102,13 @@ struct G1UpdateRegionLivenessAndSelectForRebuildTask::G1OnRegionClosure : public || hr->has_pinned_objects(); if (is_live) { const bool selected_for_rebuild = tracker->update_humongous_before_rebuild(hr); + auto on_humongous_region = [&] (G1HeapRegion* hr) { if (selected_for_rebuild) { - _num_selected_for_rebuild++; + _num_humongous_selected_for_rebuild++; } _cm->update_top_at_rebuild_start(hr); }; - _g1h->humongous_obj_regions_iterate(hr, on_humongous_region); } else { reclaim_empty_humongous_region(hr); @@ -118,7 +122,7 @@ struct G1UpdateRegionLivenessAndSelectForRebuildTask::G1OnRegionClosure : public if (is_live) { const bool selected_for_rebuild = tracker->update_old_before_rebuild(hr); if (selected_for_rebuild) { - _num_selected_for_rebuild++; + _old_selected_for_rebuild.push(hr); } _cm->update_top_at_rebuild_start(hr); } else { @@ -137,7 +141,8 @@ G1UpdateRegionLivenessAndSelectForRebuildTask::G1UpdateRegionLivenessAndSelectFo _g1h(g1h), _cm(cm), _hrclaimer(num_workers), - _total_selected_for_rebuild(0), + _old_selected_for_rebuild(128), + _num_humongous_selected_for_rebuild(0), _cleanup_list("Empty Regions After Mark List") {} G1UpdateRegionLivenessAndSelectForRebuildTask::~G1UpdateRegionLivenessAndSelectForRebuildTask() { @@ -153,8 +158,6 @@ void G1UpdateRegionLivenessAndSelectForRebuildTask::work(uint worker_id) { G1OnRegionClosure on_region_cl(_g1h, _cm, &local_cleanup_list); _g1h->heap_region_par_iterate_from_worker_offset(&on_region_cl, &_hrclaimer, worker_id); - _total_selected_for_rebuild.add_then_fetch(on_region_cl._num_selected_for_rebuild); - // Update the old/humongous region sets _g1h->remove_from_old_gen_sets(on_region_cl._num_old_regions_removed, on_region_cl._num_humongous_regions_removed); @@ -163,6 +166,9 @@ void G1UpdateRegionLivenessAndSelectForRebuildTask::work(uint worker_id) { MutexLocker x(G1RareEvent_lock, Mutex::_no_safepoint_check_flag); _g1h->decrement_summary_bytes(on_region_cl._freed_bytes); + _old_selected_for_rebuild.appendAll(&on_region_cl._old_selected_for_rebuild); + _num_humongous_selected_for_rebuild += on_region_cl._num_humongous_selected_for_rebuild; + _cleanup_list.add_ordered(&local_cleanup_list); assert(local_cleanup_list.is_empty(), "post-condition"); } @@ -172,3 +178,78 @@ uint G1UpdateRegionLivenessAndSelectForRebuildTask::desired_num_workers(uint num const uint num_regions_per_worker = 384; return (num_regions + num_regions_per_worker - 1) / num_regions_per_worker; } + +// Early prune (remove) regions meeting the G1HeapWastePercent criteria. That +// is, either until only the minimum amount of old collection set regions are +// available (for forward progress in evacuation) or the waste accumulated by the +// removed regions is above the maximum allowed waste. +// Updates number of candidates and reclaimable bytes given. +void G1UpdateRegionLivenessAndSelectForRebuildTask::prune(GrowableArrayCHeap* old_regions) { + G1Policy* p = G1CollectedHeap::heap()->policy(); + + uint num_candidates = (uint)old_regions->length(); + + uint min_old_cset_length = p->calc_min_old_cset_length(num_candidates); + uint num_pruned = 0; + size_t wasted_bytes = 0; + + if (min_old_cset_length >= num_candidates) { + // We take all of the candidate regions to provide some forward progress. + return; + } + + size_t allowed_waste = p->allowed_waste_in_collection_set(); + uint max_to_prune = num_candidates - min_old_cset_length; + + while (true) { + G1HeapRegion* r = old_regions->at(num_candidates - num_pruned - 1); + size_t const reclaimable = r->reclaimable_bytes(); + if (num_pruned >= max_to_prune || + wasted_bytes + reclaimable > allowed_waste) { + break; + } + r->rem_set()->clear(true /* cardset_only */); + + wasted_bytes += reclaimable; + num_pruned++; + } + + log_debug(gc, ergo, cset)("Pruned %u regions out of %u, leaving %zu bytes waste (allowed %zu)", + num_pruned, + num_candidates, + wasted_bytes, + allowed_waste); + + old_regions->trunc_to(num_candidates - num_pruned); +} + +static int compare_region_gc_efficiency(G1HeapRegion** rr1, G1HeapRegion** rr2) { + G1HeapRegion* r1 = *rr1; + G1HeapRegion* r2 = *rr2; + + assert(r1 != nullptr, "must be"); + assert(r2 != nullptr, "must be"); + + G1Policy* p = G1CollectedHeap::heap()->policy(); + double gc_efficiency1 = p->predict_gc_efficiency(r1); + double gc_efficiency2 = p->predict_gc_efficiency(r2); + + if (gc_efficiency1 > gc_efficiency2) { + return -1; + } else if (gc_efficiency1 < gc_efficiency2) { + return 1; + } else { + return 0; + } +} + +GrowableArrayCHeap* G1UpdateRegionLivenessAndSelectForRebuildTask::sort_and_prune_old_selected() { + // Nothing to do for the humongous candidates here. Old selected need to be pruned. + + if (_old_selected_for_rebuild.length() != 0) { + _old_selected_for_rebuild.sort(compare_region_gc_efficiency); + prune(&_old_selected_for_rebuild); + } + + return &_old_selected_for_rebuild; +} diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.hpp index a256693ff1d..6905419e2cc 100644 --- a/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.hpp +++ b/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.hpp @@ -29,7 +29,7 @@ #include "gc/g1/g1HeapRegionManager.hpp" #include "gc/g1/g1HeapRegionSet.hpp" #include "gc/shared/workerThread.hpp" -#include "runtime/atomic.hpp" +#include "utilities/growableArray.hpp" class G1CollectedHeap; class G1ConcurrentMark; @@ -42,13 +42,15 @@ class G1UpdateRegionLivenessAndSelectForRebuildTask : public WorkerTask { G1ConcurrentMark* _cm; G1HeapRegionClaimer _hrclaimer; - Atomic _total_selected_for_rebuild; + GrowableArrayCHeap _old_selected_for_rebuild; + uint _num_humongous_selected_for_rebuild; // Reclaimed empty regions G1FreeRegionList _cleanup_list; struct G1OnRegionClosure; + void prune(GrowableArrayCHeap* old_regions); public: G1UpdateRegionLivenessAndSelectForRebuildTask(G1CollectedHeap* g1h, G1ConcurrentMark* cm, @@ -59,9 +61,14 @@ public: void work(uint worker_id) override; uint total_selected_for_rebuild() const { - return _total_selected_for_rebuild.load_relaxed(); + return (uint)_old_selected_for_rebuild.length() + _num_humongous_selected_for_rebuild; } + // Sort selected old regions by efficiency and prune them based on G1HeapWastePercent. + // This pruning improves rebuild time in addition to remembered set memory usage. + // Returns the set of regions selected in efficiency order. + GrowableArrayCHeap* sort_and_prune_old_selected(); + static uint desired_num_workers(uint num_regions); }; diff --git a/src/hotspot/share/gc/g1/g1Policy.cpp b/src/hotspot/share/gc/g1/g1Policy.cpp index 1fa5aa877c0..09043c92722 100644 --- a/src/hotspot/share/gc/g1/g1Policy.cpp +++ b/src/hotspot/share/gc/g1/g1Policy.cpp @@ -28,7 +28,6 @@ #include "gc/g1/g1CollectedHeap.inline.hpp" #include "gc/g1/g1CollectionSet.hpp" #include "gc/g1/g1CollectionSetCandidates.inline.hpp" -#include "gc/g1/g1CollectionSetChooser.hpp" #include "gc/g1/g1ConcurrentMark.hpp" #include "gc/g1/g1ConcurrentMarkThread.inline.hpp" #include "gc/g1/g1ConcurrentRefine.hpp" diff --git a/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp index 0c9973c520d..94f5466b8e0 100644 --- a/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp +++ b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,12 +23,16 @@ */ #include "gc/g1/g1CollectedHeap.inline.hpp" -#include "gc/g1/g1CollectionSetChooser.hpp" #include "gc/g1/g1HeapRegion.inline.hpp" #include "gc/g1/g1HeapRegionRemSet.inline.hpp" #include "gc/g1/g1RemSetTrackingPolicy.hpp" #include "runtime/safepoint.hpp" +static bool region_occupancy_low_enough_for_evac(size_t live_bytes) { + size_t mixed_gc_live_threshold_bytes = G1HeapRegion::GrainBytes * (size_t)G1MixedGCLiveThresholdPercent / 100; + return live_bytes < mixed_gc_live_threshold_bytes; +} + void G1RemSetTrackingPolicy::update_at_allocate(G1HeapRegion* r) { assert(r->is_young() || r->is_humongous() || r->is_old(), "Region %u with unexpected heap region type %s", r->hrm_index(), r->get_type_str()); @@ -75,7 +79,8 @@ bool G1RemSetTrackingPolicy::update_old_before_rebuild(G1HeapRegion* r) { bool selected_for_rebuild = false; - if (G1CollectionSetChooser::region_occupancy_low_enough_for_evac(r->live_bytes()) && + if (region_occupancy_low_enough_for_evac(r->live_bytes()) && + !G1CollectedHeap::heap()->is_old_gc_alloc_region(r) && !r->rem_set()->is_tracked()) { r->rem_set()->set_state_updating(); selected_for_rebuild = true;