From c760097943d0fc893433d813502e2ff02bb4c82d Mon Sep 17 00:00:00 2001 From: Thomas Schatzl Date: Tue, 7 Nov 2023 13:48:18 +0000 Subject: [PATCH] 8319541: G1: Inline G1RemoveSelfForwardsTask into RestoreRetainedRegionsTask Reviewed-by: ayang, iwalulya --- src/hotspot/share/gc/g1/g1EvacFailure.cpp | 218 ------------------ src/hotspot/share/gc/g1/g1EvacFailure.hpp | 61 ----- .../gc/g1/g1YoungGCPostEvacuateTasks.cpp | 198 +++++++++++++++- .../gc/g1/g1YoungGCPostEvacuateTasks.hpp | 1 - 4 files changed, 193 insertions(+), 285 deletions(-) delete mode 100644 src/hotspot/share/gc/g1/g1EvacFailure.cpp delete mode 100644 src/hotspot/share/gc/g1/g1EvacFailure.hpp diff --git a/src/hotspot/share/gc/g1/g1EvacFailure.cpp b/src/hotspot/share/gc/g1/g1EvacFailure.cpp deleted file mode 100644 index 6eeb5d084fa..00000000000 --- a/src/hotspot/share/gc/g1/g1EvacFailure.cpp +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "gc/g1/g1CollectedHeap.inline.hpp" -#include "gc/g1/g1CollectorState.hpp" -#include "gc/g1/g1ConcurrentMark.inline.hpp" -#include "gc/g1/g1EvacFailure.hpp" -#include "gc/g1/g1EvacFailureRegions.hpp" -#include "gc/g1/g1GCPhaseTimes.hpp" -#include "gc/g1/g1OopClosures.inline.hpp" -#include "gc/g1/heapRegion.inline.hpp" -#include "gc/g1/heapRegionRemSet.inline.hpp" -#include "oops/access.inline.hpp" -#include "oops/compressedOops.inline.hpp" -#include "oops/oop.inline.hpp" -#include "runtime/prefetch.hpp" -#include "utilities/bitMap.inline.hpp" - - -class PhaseTimesStat { - static constexpr G1GCPhaseTimes::GCParPhases phase_name = - G1GCPhaseTimes::RemoveSelfForwards; - - G1GCPhaseTimes* _phase_times; - uint _worker_id; - Ticks _start; - -public: - PhaseTimesStat(G1GCPhaseTimes* phase_times, uint worker_id) : - _phase_times(phase_times), - _worker_id(worker_id), - _start(Ticks::now()) { } - - ~PhaseTimesStat() { - _phase_times->record_or_add_time_secs(phase_name, - _worker_id, - (Ticks::now() - _start).seconds()); - } - - void register_empty_chunk() { - _phase_times->record_or_add_thread_work_item(phase_name, - _worker_id, - 1, - G1GCPhaseTimes::RemoveSelfForwardEmptyChunksNum); - } - - void register_nonempty_chunk() { - _phase_times->record_or_add_thread_work_item(phase_name, - _worker_id, - 1, - G1GCPhaseTimes::RemoveSelfForwardChunksNum); - } - - void register_objects_count_and_size(size_t num_marked_obj, size_t marked_words) { - _phase_times->record_or_add_thread_work_item(phase_name, - _worker_id, - num_marked_obj, - G1GCPhaseTimes::RemoveSelfForwardObjectsNum); - - size_t marked_bytes = marked_words * HeapWordSize; - _phase_times->record_or_add_thread_work_item(phase_name, - _worker_id, - marked_bytes, - G1GCPhaseTimes::RemoveSelfForwardObjectsBytes); - } -}; - -// Fill the memory area from start to end with filler objects, and update the BOT -// accordingly. Since we clear and use the bitmap for marking objects that failed -// evacuation, there is no other work to be done there. -static size_t zap_dead_objects(HeapRegion* hr, HeapWord* start, HeapWord* end) { - assert(start <= end, "precondition"); - if (start == end) { - return 0; - } - - hr->fill_range_with_dead_objects(start, end); - return pointer_delta(end, start); -} - -static void update_garbage_words_in_hr(HeapRegion* hr, size_t garbage_words) { - if (garbage_words != 0) { - hr->note_self_forward_chunk_done(garbage_words * HeapWordSize); - } -} - -static void prefetch_obj(HeapWord* obj_addr) { - Prefetch::write(obj_addr, PrefetchScanIntervalInBytes); -} - -void G1RemoveSelfForwardsTask::process_chunk(uint worker_id, - uint chunk_idx) { - PhaseTimesStat stat(_g1h->phase_times(), worker_id); - - G1CMBitMap* bitmap = _cm->mark_bitmap(); - const uint region_idx = _evac_failure_regions->get_region_idx(chunk_idx / _num_chunks_per_region); - HeapRegion* hr = _g1h->region_at(region_idx); - - HeapWord* hr_bottom = hr->bottom(); - HeapWord* hr_top = hr->top(); - HeapWord* chunk_start = hr_bottom + (chunk_idx % _num_chunks_per_region) * _chunk_size; - - assert(chunk_start < hr->end(), "inv"); - if (chunk_start >= hr_top) { - return; - } - - HeapWord* chunk_end = MIN2(chunk_start + _chunk_size, hr_top); - HeapWord* first_marked_addr = bitmap->get_next_marked_addr(chunk_start, hr_top); - - size_t garbage_words = 0; - - if (chunk_start == hr_bottom) { - // This is the bottom-most chunk in this region; zap [bottom, first_marked_addr). - garbage_words += zap_dead_objects(hr, hr_bottom, first_marked_addr); - } - - if (first_marked_addr >= chunk_end) { - stat.register_empty_chunk(); - update_garbage_words_in_hr(hr, garbage_words); - return; - } - - stat.register_nonempty_chunk(); - - size_t num_marked_objs = 0; - size_t marked_words = 0; - - HeapWord* obj_addr = first_marked_addr; - assert(chunk_start <= obj_addr && obj_addr < chunk_end, - "object " PTR_FORMAT " must be within chunk [" PTR_FORMAT ", " PTR_FORMAT "[", - p2i(obj_addr), p2i(chunk_start), p2i(chunk_end)); - do { - assert(bitmap->is_marked(obj_addr), "inv"); - prefetch_obj(obj_addr); - - oop obj = cast_to_oop(obj_addr); - const size_t obj_size = obj->size(); - HeapWord* const obj_end_addr = obj_addr + obj_size; - - { - // Process marked object. - assert(obj->is_forwarded() && obj->forwardee() == obj, "must be self-forwarded"); - obj->init_mark(); - hr->update_bot_for_block(obj_addr, obj_end_addr); - - // Statistics - num_marked_objs++; - marked_words += obj_size; - } - - assert(obj_end_addr <= hr_top, "inv"); - // Use hr_top as the limit so that we zap dead ranges up to the next - // marked obj or hr_top. - HeapWord* next_marked_obj_addr = bitmap->get_next_marked_addr(obj_end_addr, hr_top); - garbage_words += zap_dead_objects(hr, obj_end_addr, next_marked_obj_addr); - obj_addr = next_marked_obj_addr; - } while (obj_addr < chunk_end); - - assert(marked_words > 0 && num_marked_objs > 0, "inv"); - - stat.register_objects_count_and_size(num_marked_objs, marked_words); - - update_garbage_words_in_hr(hr, garbage_words); -} - -G1RemoveSelfForwardsTask::G1RemoveSelfForwardsTask(G1EvacFailureRegions* evac_failure_regions) : - WorkerTask("G1 Remove Self-forwarding Pointers"), - _g1h(G1CollectedHeap::heap()), - _cm(_g1h->concurrent_mark()), - _evac_failure_regions(evac_failure_regions), - _chunk_bitmap(mtGC) { - - _num_evac_fail_regions = _evac_failure_regions->num_regions_failed_evacuation(); - _num_chunks_per_region = G1CollectedHeap::get_chunks_per_region(); - - _chunk_size = static_cast(HeapRegion::GrainWords / _num_chunks_per_region); - - log_debug(gc, ergo)("Initializing removing self forwards with %u chunks per region", - _num_chunks_per_region); - - _chunk_bitmap.resize(_num_chunks_per_region * _num_evac_fail_regions); -} - -void G1RemoveSelfForwardsTask::work(uint worker_id) { - const uint total_workers = G1CollectedHeap::heap()->workers()->active_workers(); - const uint total_chunks = _num_chunks_per_region * _num_evac_fail_regions; - const uint start_chunk_idx = worker_id * total_chunks / total_workers; - - for (uint i = 0; i < total_chunks; i++) { - const uint chunk_idx = (start_chunk_idx + i) % total_chunks; - if (claim_chunk(chunk_idx)) { - process_chunk(worker_id, chunk_idx); - } - } -} diff --git a/src/hotspot/share/gc/g1/g1EvacFailure.hpp b/src/hotspot/share/gc/g1/g1EvacFailure.hpp deleted file mode 100644 index d54edb8231a..00000000000 --- a/src/hotspot/share/gc/g1/g1EvacFailure.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_GC_G1_G1EVACFAILURE_HPP -#define SHARE_GC_G1_G1EVACFAILURE_HPP - -#include "gc/shared/workerThread.hpp" -#include "utilities/globalDefinitions.hpp" -#include "utilities/bitMap.hpp" - -class G1CollectedHeap; -class G1ConcurrentMark; -class G1EvacFailureRegions; - -// Task to fixup self-forwarding pointers within the objects installed as a result -// of an evacuation failure. -class G1RemoveSelfForwardsTask : public WorkerTask { - G1CollectedHeap* _g1h; - G1ConcurrentMark* _cm; - - G1EvacFailureRegions* _evac_failure_regions; - CHeapBitMap _chunk_bitmap; - - uint _num_chunks_per_region; - uint _num_evac_fail_regions; - size_t _chunk_size; - - bool claim_chunk(uint chunk_idx) { - return _chunk_bitmap.par_set_bit(chunk_idx); - } - - void process_chunk(uint worker_id, uint chunk_idx); - -public: - explicit G1RemoveSelfForwardsTask(G1EvacFailureRegions* evac_failure_regions); - - void work(uint worker_id); -}; - -#endif // SHARE_GC_G1_G1EVACFAILURE_HPP diff --git a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp index 02820869148..6e1d1cdd3be 100644 --- a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp +++ b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp @@ -29,16 +29,26 @@ #include "gc/g1/g1CardTableEntryClosure.hpp" #include "gc/g1/g1CollectedHeap.inline.hpp" #include "gc/g1/g1CollectionSetCandidates.inline.hpp" +#include "gc/g1/g1CollectorState.hpp" #include "gc/g1/g1ConcurrentMark.inline.hpp" -#include "gc/g1/g1EvacStats.inline.hpp" +#include "gc/g1/g1EvacFailureRegions.hpp" #include "gc/g1/g1EvacInfo.hpp" +#include "gc/g1/g1EvacStats.inline.hpp" +#include "gc/g1/g1OopClosures.inline.hpp" #include "gc/g1/g1ParScanThreadState.hpp" #include "gc/g1/g1RemSet.hpp" #include "gc/g1/g1YoungGCPostEvacuateTasks.hpp" +#include "gc/g1/heapRegion.inline.hpp" +#include "gc/g1/heapRegionRemSet.inline.hpp" #include "gc/shared/preservedMarks.inline.hpp" #include "jfr/jfrEvents.hpp" +#include "oops/access.inline.hpp" +#include "oops/compressedOops.inline.hpp" +#include "oops/oop.inline.hpp" +#include "runtime/prefetch.hpp" #include "runtime/threads.hpp" #include "runtime/threadSMR.hpp" +#include "utilities/bitMap.inline.hpp" #include "utilities/ticks.hpp" class G1PostEvacuateCollectionSetCleanupTask1::MergePssTask : public G1AbstractSubTask { @@ -93,14 +103,183 @@ public: }; class G1PostEvacuateCollectionSetCleanupTask1::RestoreRetainedRegionsTask : public G1AbstractSubTask { - G1RemoveSelfForwardsTask _task; + G1CollectedHeap* _g1h; + G1ConcurrentMark* _cm; + G1EvacFailureRegions* _evac_failure_regions; + CHeapBitMap _chunk_bitmap; + + uint _num_chunks_per_region; + uint _num_evac_fail_regions; + size_t _chunk_size; + + class PhaseTimesStat { + static constexpr G1GCPhaseTimes::GCParPhases phase_name = + G1GCPhaseTimes::RemoveSelfForwards; + + G1GCPhaseTimes* _phase_times; + uint _worker_id; + Ticks _start; + + public: + PhaseTimesStat(G1GCPhaseTimes* phase_times, uint worker_id) : + _phase_times(phase_times), + _worker_id(worker_id), + _start(Ticks::now()) { } + + ~PhaseTimesStat() { + _phase_times->record_or_add_time_secs(phase_name, + _worker_id, + (Ticks::now() - _start).seconds()); + } + + void register_empty_chunk() { + _phase_times->record_or_add_thread_work_item(phase_name, + _worker_id, + 1, + G1GCPhaseTimes::RemoveSelfForwardEmptyChunksNum); + } + + void register_nonempty_chunk() { + _phase_times->record_or_add_thread_work_item(phase_name, + _worker_id, + 1, + G1GCPhaseTimes::RemoveSelfForwardChunksNum); + } + + void register_objects_count_and_size(size_t num_marked_obj, size_t marked_words) { + _phase_times->record_or_add_thread_work_item(phase_name, + _worker_id, + num_marked_obj, + G1GCPhaseTimes::RemoveSelfForwardObjectsNum); + + size_t marked_bytes = marked_words * HeapWordSize; + _phase_times->record_or_add_thread_work_item(phase_name, + _worker_id, + marked_bytes, + G1GCPhaseTimes::RemoveSelfForwardObjectsBytes); + } + }; + + // Fill the memory area from start to end with filler objects, and update the BOT + // accordingly. Since we clear and use the bitmap for marking objects that failed + // evacuation, there is no other work to be done there. + static size_t zap_dead_objects(HeapRegion* hr, HeapWord* start, HeapWord* end) { + assert(start <= end, "precondition"); + if (start == end) { + return 0; + } + + hr->fill_range_with_dead_objects(start, end); + return pointer_delta(end, start); + } + + static void update_garbage_words_in_hr(HeapRegion* hr, size_t garbage_words) { + if (garbage_words != 0) { + hr->note_self_forward_chunk_done(garbage_words * HeapWordSize); + } + } + + static void prefetch_obj(HeapWord* obj_addr) { + Prefetch::write(obj_addr, PrefetchScanIntervalInBytes); + } + + bool claim_chunk(uint chunk_idx) { + return _chunk_bitmap.par_set_bit(chunk_idx); + } + + void process_chunk(uint worker_id, uint chunk_idx) { + PhaseTimesStat stat(_g1h->phase_times(), worker_id); + + G1CMBitMap* bitmap = _cm->mark_bitmap(); + const uint region_idx = _evac_failure_regions->get_region_idx(chunk_idx / _num_chunks_per_region); + HeapRegion* hr = _g1h->region_at(region_idx); + + HeapWord* hr_bottom = hr->bottom(); + HeapWord* hr_top = hr->top(); + HeapWord* chunk_start = hr_bottom + (chunk_idx % _num_chunks_per_region) * _chunk_size; + + assert(chunk_start < hr->end(), "inv"); + if (chunk_start >= hr_top) { + return; + } + + HeapWord* chunk_end = MIN2(chunk_start + _chunk_size, hr_top); + HeapWord* first_marked_addr = bitmap->get_next_marked_addr(chunk_start, hr_top); + + size_t garbage_words = 0; + + if (chunk_start == hr_bottom) { + // This is the bottom-most chunk in this region; zap [bottom, first_marked_addr). + garbage_words += zap_dead_objects(hr, hr_bottom, first_marked_addr); + } + + if (first_marked_addr >= chunk_end) { + stat.register_empty_chunk(); + update_garbage_words_in_hr(hr, garbage_words); + return; + } + + stat.register_nonempty_chunk(); + + size_t num_marked_objs = 0; + size_t marked_words = 0; + + HeapWord* obj_addr = first_marked_addr; + assert(chunk_start <= obj_addr && obj_addr < chunk_end, + "object " PTR_FORMAT " must be within chunk [" PTR_FORMAT ", " PTR_FORMAT "[", + p2i(obj_addr), p2i(chunk_start), p2i(chunk_end)); + do { + assert(bitmap->is_marked(obj_addr), "inv"); + prefetch_obj(obj_addr); + + oop obj = cast_to_oop(obj_addr); + const size_t obj_size = obj->size(); + HeapWord* const obj_end_addr = obj_addr + obj_size; + + { + // Process marked object. + assert(obj->is_forwarded() && obj->forwardee() == obj, "must be self-forwarded"); + obj->init_mark(); + hr->update_bot_for_block(obj_addr, obj_end_addr); + + // Statistics + num_marked_objs++; + marked_words += obj_size; + } + + assert(obj_end_addr <= hr_top, "inv"); + // Use hr_top as the limit so that we zap dead ranges up to the next + // marked obj or hr_top. + HeapWord* next_marked_obj_addr = bitmap->get_next_marked_addr(obj_end_addr, hr_top); + garbage_words += zap_dead_objects(hr, obj_end_addr, next_marked_obj_addr); + obj_addr = next_marked_obj_addr; + } while (obj_addr < chunk_end); + + assert(marked_words > 0 && num_marked_objs > 0, "inv"); + + stat.register_objects_count_and_size(num_marked_objs, marked_words); + + update_garbage_words_in_hr(hr, garbage_words); + } public: RestoreRetainedRegionsTask(G1EvacFailureRegions* evac_failure_regions) : G1AbstractSubTask(G1GCPhaseTimes::RestoreRetainedRegions), - _task(evac_failure_regions), - _evac_failure_regions(evac_failure_regions) { + _g1h(G1CollectedHeap::heap()), + _cm(_g1h->concurrent_mark()), + _evac_failure_regions(evac_failure_regions), + _chunk_bitmap(mtGC) { + + _num_evac_fail_regions = _evac_failure_regions->num_regions_failed_evacuation(); + _num_chunks_per_region = G1CollectedHeap::get_chunks_per_region(); + + _chunk_size = static_cast(HeapRegion::GrainWords / _num_chunks_per_region); + + log_debug(gc, ergo)("Initializing removing self forwards with %u chunks per region", + _num_chunks_per_region); + + _chunk_bitmap.resize(_num_chunks_per_region * _num_evac_fail_regions); } double worker_cost() const override { @@ -111,7 +290,16 @@ public: } void do_work(uint worker_id) override { - _task.work(worker_id); + const uint total_workers = G1CollectedHeap::heap()->workers()->active_workers(); + const uint total_chunks = _num_chunks_per_region * _num_evac_fail_regions; + const uint start_chunk_idx = worker_id * total_chunks / total_workers; + + for (uint i = 0; i < total_chunks; i++) { + const uint chunk_idx = (start_chunk_idx + i) % total_chunks; + if (claim_chunk(chunk_idx)) { + process_chunk(worker_id, chunk_idx); + } + } } }; diff --git a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp index 07d62ac04d5..511eb1b4afb 100644 --- a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp +++ b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp @@ -26,7 +26,6 @@ #define SHARE_GC_G1_G1YOUNGGCPOSTEVACUATETASKS_HPP #include "gc/g1/g1BatchedTask.hpp" -#include "gc/g1/g1EvacFailure.hpp" class FreeCSetStats;