infernap12/jdk
1
0
Fork 0
mirror of https://github.com/openjdk/jdk.git synced 2026-03-17 03:13:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

8254167: G1: Record regions where evacuation failed to provide targeted iteration

Browse Source 
Reviewed-by: tschatzl, ayang
This commit is contained in:
Hamlin Li 2021-09-08 08:01:18 +00:00
parent 286a1f6b7c
commit a66629a464
16 changed files with 351 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
src/hotspot/share/gc/g1/g1CollectedHeap.cpp
View File

@ -1474,8 +1474,6 @@ G1CollectedHeap::G1CollectedHeap() :
_cm_thread(NULL),
_cr(NULL),
_task_queues(NULL),
_num_regions_failed_evacuation(0),
_regions_failed_evacuation(mtGC),
_ref_processor_stw(NULL),
_is_alive_closure_stw(this),
_is_subject_to_discovery_stw(this),
@ -1756,7 +1754,7 @@ jint G1CollectedHeap::initialize() {
_collection_set.initialize(max_reserved_regions());
_regions_failed_evacuation.resize(max_regions());
_evac_failure_regions.initialize(max_reserved_regions());
evac_failure_injector()->reset();
@ -2316,12 +2314,46 @@ void G1CollectedHeap::collection_set_iterate_all(HeapRegionClosure* cl) {
_collection_set.iterate(cl);
}
void G1CollectedHeap::collection_set_par_iterate_all(HeapRegionClosure* cl, HeapRegionClaimer* hr_claimer, uint worker_id) {
_collection_set.par_iterate(cl, hr_claimer, worker_id, workers()->active_workers());
void G1CollectedHeap::collection_set_par_iterate_all(HeapRegionClosure* cl,
HeapRegionClaimer* hr_claimer,
uint worker_id) {
_collection_set.par_iterate(cl, hr_claimer, worker_id);
}
void G1CollectedHeap::collection_set_iterate_increment_from(HeapRegionClosure *cl, HeapRegionClaimer* hr_claimer, uint worker_id) {
_collection_set.iterate_incremental_part_from(cl, hr_claimer, worker_id, workers()->active_workers());
void G1CollectedHeap::collection_set_iterate_increment_from(HeapRegionClosure *cl,
HeapRegionClaimer* hr_claimer,
uint worker_id) {
_collection_set.iterate_incremental_part_from(cl, hr_claimer, worker_id);
}
void G1CollectedHeap::par_iterate_regions_array_part_from(HeapRegionClosure* cl,
HeapRegionClaimer* hr_claimer,
const uint* regions,
size_t offset,
size_t length,
uint worker_id) const {
assert_at_safepoint();
if (length == 0) {
return;
}
uint total_workers = workers()->active_workers();
size_t start_pos = (worker_id * length) / total_workers;
size_t cur_pos = start_pos;
do {
uint region_idx = regions[cur_pos + offset];
if (hr_claimer == NULL || hr_claimer->claim_region(region_idx)) {
HeapRegion* r = region_at(region_idx);
bool result = cl->do_heap_region(r);
guarantee(!result, "Must not cancel iteration");
}
cur_pos++;
if (cur_pos == length) {
cur_pos = 0;
}
} while (cur_pos != start_pos);
}
HeapWord* G1CollectedHeap::block_start(const void* addr) const {
@ -2855,7 +2887,7 @@ void G1CollectedHeap::do_collection_pause_at_safepoint_helper(double target_paus
bool should_start_concurrent_mark_operation = collector_state()->in_concurrent_start_gc();
// Perform the collection.
G1YoungCollector collector(gc_cause(), target_pause_time_ms);
G1YoungCollector collector(gc_cause(), target_pause_time_ms, &_evac_failure_regions);
collector.collect();
// It should now be safe to tell the concurrent mark thread to start

25
src/hotspot/share/gc/g1/g1CollectedHeap.hpp
View File

@ -34,6 +34,7 @@
#include "gc/g1/g1ConcurrentMark.hpp"
#include "gc/g1/g1EdenRegions.hpp"
#include "gc/g1/g1EvacStats.hpp"
#include "gc/g1/g1EvacFailureRegions.hpp"
#include "gc/g1/g1GCPauseType.hpp"
#include "gc/g1/g1HeapTransition.hpp"
#include "gc/g1/g1HeapVerifier.hpp"
@ -830,10 +831,7 @@ public:
// The parallel task queues
G1ScannerTasksQueueSet *_task_queues;
// Number of regions evacuation failed in the current collection.
volatile uint _num_regions_failed_evacuation;
// Records for every region on the heap whether evacuation failed for it.
CHeapBitMap _regions_failed_evacuation;
G1EvacFailureRegions _evac_failure_regions;
// ("Weak") Reference processing support.
//
@ -1052,18 +1050,8 @@ public:
void start_concurrent_gc_for_metadata_allocation(GCCause::Cause gc_cause);
inline void reset_evacuation_failed_data();
// True iff an evacuation has failed in the most-recent collection.
inline bool evacuation_failed() const;
// True iff the given region encountered an evacuation failure in the most-recent
// collection.
inline bool evacuation_failed(uint region_idx) const;
inline uint num_regions_failed_evacuation() const;
// Notify that the garbage collection encountered an evacuation failure in the
// given region. Returns whether this has been the first occurrence of an evacuation
// failure in that region.
inline bool notify_region_failed_evacuation(uint const region_idx);
void remove_from_old_gen_sets(const uint old_regions_removed,
const uint archive_regions_removed,
@ -1167,6 +1155,15 @@ public:
collection_set_iterate_increment_from(blk, NULL, worker_id);
}
void collection_set_iterate_increment_from(HeapRegionClosure *blk, HeapRegionClaimer* hr_claimer, uint worker_id);
// Iterate part of an array of region indexes given by offset and length, applying
// the given HeapRegionClosure on each region. The worker_id will determine where
// in the part to start the iteration to allow for more efficient parallel iteration.
void par_iterate_regions_array_part_from(HeapRegionClosure* cl,
HeapRegionClaimer* hr_claimer,
const uint* regions,
size_t offset,
size_t length,
uint worker_id) const;
// Returns the HeapRegion that contains addr. addr must not be NULL.
template <class T>

24
src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp
View File

@ -29,6 +29,7 @@
#include "gc/g1/g1BarrierSet.hpp"
#include "gc/g1/g1CollectorState.hpp"
#include "gc/g1/g1EvacFailureRegions.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1RemSet.hpp"
#include "gc/g1/heapRegionManager.inline.hpp"
@ -195,29 +196,8 @@ void G1CollectedHeap::register_optional_region_with_region_attr(HeapRegion* r) {
_region_attr.set_optional(r->hrm_index(), r->rem_set()->is_tracked());
}
void G1CollectedHeap::reset_evacuation_failed_data() {
Atomic::store(&_num_regions_failed_evacuation, 0u);
_regions_failed_evacuation.clear();
}
bool G1CollectedHeap::evacuation_failed() const {
return num_regions_failed_evacuation() > 0;
}
bool G1CollectedHeap::evacuation_failed(uint region_idx) const {
return _regions_failed_evacuation.par_at(region_idx, memory_order_relaxed);
}
uint G1CollectedHeap::num_regions_failed_evacuation() const {
return Atomic::load(&_num_regions_failed_evacuation);
}
bool G1CollectedHeap::notify_region_failed_evacuation(uint const region_idx) {
bool result = _regions_failed_evacuation.par_set_bit(region_idx, memory_order_relaxed);
if (result) {
Atomic::inc(&_num_regions_failed_evacuation, memory_order_relaxed);
}
return result;
return _evac_failure_regions.num_regions_failed_evacuation() > 0;
}
inline bool G1CollectedHeap::is_in_young(const oop obj) {

35
src/hotspot/share/gc/g1/g1CollectionSet.cpp
View File

@ -207,9 +207,8 @@ void G1CollectionSet::iterate(HeapRegionClosure* cl) const {
void G1CollectionSet::par_iterate(HeapRegionClosure* cl,
HeapRegionClaimer* hr_claimer,
uint worker_id,
uint total_workers) const {
iterate_part_from(cl, hr_claimer, 0, cur_length(), worker_id, total_workers);
uint worker_id) const {
iterate_part_from(cl, hr_claimer, 0, cur_length(), worker_id);
}
void G1CollectionSet::iterate_optional(HeapRegionClosure* cl) const {
@ -224,38 +223,16 @@ void G1CollectionSet::iterate_optional(HeapRegionClosure* cl) const {
void G1CollectionSet::iterate_incremental_part_from(HeapRegionClosure* cl,
HeapRegionClaimer* hr_claimer,
uint worker_id,
uint total_workers) const {
iterate_part_from(cl, hr_claimer, _inc_part_start, increment_length(), worker_id, total_workers);
uint worker_id) const {
iterate_part_from(cl, hr_claimer, _inc_part_start, increment_length(), worker_id);
}
void G1CollectionSet::iterate_part_from(HeapRegionClosure* cl,
HeapRegionClaimer* hr_claimer,
size_t offset,
size_t length,
uint worker_id,
uint total_workers) const {
assert_at_safepoint();
if (length == 0) {
return;
}
size_t start_pos = (worker_id * length) / total_workers;
size_t cur_pos = start_pos;
do {
uint region_idx = _collection_set_regions[cur_pos + offset];
if (hr_claimer == NULL || hr_claimer->claim_region(region_idx)) {
HeapRegion* r = _g1h->region_at(region_idx);
bool result = cl->do_heap_region(r);
guarantee(!result, "Must not cancel iteration");
}
cur_pos++;
if (cur_pos == length) {
cur_pos = 0;
}
} while (cur_pos != start_pos);
uint worker_id) const {
_g1h->par_iterate_regions_array_part_from(cl, hr_claimer, _collection_set_regions, offset, length, worker_id);
}
void G1CollectionSet::update_young_region_prediction(HeapRegion* hr,

8
src/hotspot/share/gc/g1/g1CollectionSet.hpp
View File

@ -262,8 +262,7 @@ class G1CollectionSet {
HeapRegionClaimer* hr_claimer,
size_t offset,
size_t length,
uint worker_id,
uint total_workers) const;
uint worker_id) const;
public:
G1CollectionSet(G1CollectedHeap* g1h, G1Policy* policy);
~G1CollectionSet();
@ -307,7 +306,7 @@ public:
// Iterate over the current collection set increment applying the given HeapRegionClosure
// from a starting position determined by the given worker id.
void iterate_incremental_part_from(HeapRegionClosure* cl, HeapRegionClaimer* hr_claimer, uint worker_id, uint total_workers) const;
void iterate_incremental_part_from(HeapRegionClosure* cl, HeapRegionClaimer* hr_claimer, uint worker_id) const;
// Returns the length of the current increment in number of regions.
size_t increment_length() const { return _collection_set_cur_length - _inc_part_start; }
@ -319,8 +318,7 @@ public:
void iterate(HeapRegionClosure* cl) const;
void par_iterate(HeapRegionClosure* cl,
HeapRegionClaimer* hr_claimer,
uint worker_id,
uint total_workers) const;
uint worker_id) const;
void iterate_optional(HeapRegionClosure* cl) const;

26
src/hotspot/share/gc/g1/g1EvacFailure.cpp
View File

@ -27,6 +27,7 @@
#include "gc/g1/g1CollectorState.hpp"
#include "gc/g1/g1ConcurrentMark.inline.hpp"
#include "gc/g1/g1EvacFailure.hpp"
#include "gc/g1/g1EvacFailureRegions.hpp"
#include "gc/g1/g1HeapVerifier.hpp"
#include "gc/g1/g1OopClosures.inline.hpp"
#include "gc/g1/g1RedirtyCardsQueue.hpp"
@ -203,14 +204,19 @@ class RemoveSelfForwardPtrHRClosure: public HeapRegionClosure {
UpdateLogBuffersDeferred _log_buffer_cl;
uint volatile* _num_failed_regions;
G1EvacFailureRegions* _evac_failure_regions;
public:
RemoveSelfForwardPtrHRClosure(G1RedirtyCardsQueueSet* rdcqs, uint worker_id, uint volatile* num_failed_regions) :
RemoveSelfForwardPtrHRClosure(G1RedirtyCardsQueueSet* rdcqs,
uint worker_id,
uint volatile* num_failed_regions,
G1EvacFailureRegions* evac_failure_regions) :
_g1h(G1CollectedHeap::heap()),
_worker_id(worker_id),
_rdc_local_qset(rdcqs),
_log_buffer_cl(&_rdc_local_qset),
_num_failed_regions(num_failed_regions) {
_num_failed_regions(num_failed_regions),
_evac_failure_regions(evac_failure_regions) {
}
~RemoveSelfForwardPtrHRClosure() {
@ -234,7 +240,7 @@ public:
assert(!hr->is_pinned(), "Unexpected pinned region at index %u", hr->hrm_index());
assert(hr->in_collection_set(), "bad CS");
if (_g1h->evacuation_failed(hr->hrm_index())) {
if (_evac_failure_regions->contains(hr->hrm_index())) {
hr->clear_index_in_opt_cset();
bool during_concurrent_start = _g1h->collector_state()->in_concurrent_start_gc();
@ -259,22 +265,20 @@ public:
}
};
G1ParRemoveSelfForwardPtrsTask::G1ParRemoveSelfForwardPtrsTask(G1RedirtyCardsQueueSet* rdcqs) :
G1ParRemoveSelfForwardPtrsTask::G1ParRemoveSelfForwardPtrsTask(G1RedirtyCardsQueueSet* rdcqs,
G1EvacFailureRegions* evac_failure_regions) :
AbstractGangTask("G1 Remove Self-forwarding Pointers"),
_g1h(G1CollectedHeap::heap()),
_rdcqs(rdcqs),
_hrclaimer(_g1h->workers()->active_workers()),
_evac_failure_regions(evac_failure_regions),
_num_failed_regions(0) { }
void G1ParRemoveSelfForwardPtrsTask::work(uint worker_id) {
RemoveSelfForwardPtrHRClosure rsfp_cl(_rdcqs, worker_id, &_num_failed_regions);
RemoveSelfForwardPtrHRClosure rsfp_cl(_rdcqs, worker_id, &_num_failed_regions, _evac_failure_regions);
// We need to check all collection set regions whether they need self forward
// removals, not only the last collection set increment. The reason is that
// reference processing (e.g. finalizers) can make it necessary to resurrect an
// otherwise unreachable object at the very end of the collection. That object
// might cause an evacuation failure in any region in the collection set.
_g1h->collection_set_par_iterate_all(&rsfp_cl, &_hrclaimer, worker_id);
// Iterate through all regions that failed evacuation during the entire collection.
_evac_failure_regions->par_iterate(&rsfp_cl, &_hrclaimer, worker_id);
}
uint G1ParRemoveSelfForwardPtrsTask::num_failed_regions() const {

4
src/hotspot/share/gc/g1/g1EvacFailure.hpp
View File

@ -31,6 +31,7 @@
#include "utilities/globalDefinitions.hpp"
class G1CollectedHeap;
class G1EvacFailureRegions;
class G1RedirtyCardsQueueSet;
// Task to fixup self-forwarding pointers
@ -41,10 +42,11 @@ protected:
G1RedirtyCardsQueueSet* _rdcqs;
HeapRegionClaimer _hrclaimer;
G1EvacFailureRegions* _evac_failure_regions;
uint volatile _num_failed_regions;
public:
G1ParRemoveSelfForwardPtrsTask(G1RedirtyCardsQueueSet* rdcqs);
G1ParRemoveSelfForwardPtrsTask(G1RedirtyCardsQueueSet* rdcqs, G1EvacFailureRegions* evac_failure_regions);
void work(uint worker_id);

68
src/hotspot/share/gc/g1/g1EvacFailureRegions.cpp Normal file
View File

@ -0,0 +1,68 @@
/*
* Copyright (c) 2021, Huawei Technologies Co. Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "gc/g1/g1CollectedHeap.hpp"
#include "gc/g1/g1EvacFailureRegions.hpp"
#include "gc/g1/heapRegion.hpp"
#include "memory/allocation.hpp"
#include "runtime/atomic.hpp"
G1EvacFailureRegions::G1EvacFailureRegions() :
_regions_failed_evacuation(mtGC) {
}
G1EvacFailureRegions::~G1EvacFailureRegions() {
FREE_C_HEAP_ARRAY(uint, _evac_failure_regions);
}
void G1EvacFailureRegions::initialize(uint max_regions) {
Atomic::store(&_evac_failure_regions_cur_length, 0u);
_max_regions = max_regions;
_regions_failed_evacuation.resize(_max_regions);
_evac_failure_regions = NEW_C_HEAP_ARRAY(uint, _max_regions, mtGC);
}
void G1EvacFailureRegions::par_iterate(HeapRegionClosure* closure,
HeapRegionClaimer* _hrclaimer,
uint worker_id) {
G1CollectedHeap::heap()->par_iterate_regions_array_part_from(closure,
_hrclaimer,
_evac_failure_regions,
0,
Atomic::load(&_evac_failure_regions_cur_length),
worker_id);
}
void G1EvacFailureRegions::reset() {
Atomic::store(&_evac_failure_regions_cur_length, 0u);
_regions_failed_evacuation.clear();
}
bool G1EvacFailureRegions::contains(uint region_idx) const {
assert(region_idx < _max_regions, "must be");
return _regions_failed_evacuation.par_at(region_idx, memory_order_relaxed);
}

69
src/hotspot/share/gc/g1/g1EvacFailureRegions.hpp Normal file
View File

@ -0,0 +1,69 @@
/*
* Copyright (c) 2021, Huawei Technologies Co. Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_GC_G1_G1EVACFAILUREREGIONS_HPP
#define SHARE_GC_G1_G1EVACFAILUREREGIONS_HPP
#include "runtime/atomic.hpp"
#include "utilities/bitMap.hpp"
class HeapRegionClosure;
class HeapRegionClaimer;
// This class records for every region on the heap whether evacuation failed for it,
// and records for every evacuation failure region to speed up iteration of these
// regions in post evacuation phase.
class G1EvacFailureRegions {
// Records for every region on the heap whether evacuation failed for it.
CHeapBitMap _regions_failed_evacuation;
// Regions (index) of evacuation failed in the current collection.
uint* _evac_failure_regions;
// Number of regions evacuation failed in the current collection.
volatile uint _evac_failure_regions_cur_length;
// Maximum of regions number.
uint _max_regions;
public:
G1EvacFailureRegions();
~G1EvacFailureRegions();
void initialize(uint max_regions);
void reset();
bool contains(uint region_idx) const;
void par_iterate(HeapRegionClosure* closure,
HeapRegionClaimer* _hrclaimer,
uint worker_id);
uint num_regions_failed_evacuation() const {
return Atomic::load(&_evac_failure_regions_cur_length);
}
// Record that the garbage collection encountered an evacuation failure in the
// given region. Returns whether this has been the first occurrence of an evacuation
// failure in that region.
inline bool record(uint region_idx);
};
#endif //SHARE_GC_G1_G1EVACFAILUREREGIONS_HPP

43
src/hotspot/share/gc/g1/g1EvacFailureRegions.inline.hpp Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright (c) 2021, Huawei Technologies Co. Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_GC_G1_G1EVACFAILUREREGIONS_INLINE_HPP
#define SHARE_GC_G1_G1EVACFAILUREREGIONS_INLINE_HPP
#include "gc/g1/g1EvacFailureRegions.hpp"
#include "runtime/atomic.hpp"
#include "utilities/bitMap.inline.hpp"
bool G1EvacFailureRegions::record(uint region_idx) {
assert(region_idx < _max_regions, "must be");
bool success = _regions_failed_evacuation.par_set_bit(region_idx,
memory_order_relaxed);
if (success) {
size_t offset = Atomic::fetch_and_add(&_evac_failure_regions_cur_length, 1u);
_evac_failure_regions[offset] = region_idx;
}
return success;
}
#endif //SHARE_GC_G1_G1EVACFAILUREREGIONS_INLINE_HPP

18
src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
View File

@ -26,6 +26,7 @@
#include "gc/g1/g1Allocator.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1CollectionSet.hpp"
#include "gc/g1/g1EvacFailureRegions.inline.hpp"
#include "gc/g1/g1OopClosures.inline.hpp"
#include "gc/g1/g1ParScanThreadState.inline.hpp"
#include "gc/g1/g1RootClosures.hpp"
@ -58,7 +59,8 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
uint worker_id,
uint n_workers,
size_t young_cset_length,
size_t optional_cset_length)
size_t optional_cset_length,
G1EvacFailureRegions* evac_failure_regions)
: _g1h(g1h),
_task_queue(g1h->task_queue(worker_id)),
_rdc_local_qset(rdcqs),
@ -84,7 +86,8 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
_numa(g1h->numa()),
_obj_alloc_stat(NULL),
_preserved_marks(preserved_marks),
_evacuation_failed_info()
_evacuation_failed_info(),
_evac_failure_regions(evac_failure_regions)
{
// We allocate number of young gen regions in the collection set plus one
// entries, since entry 0 keeps track of surviving bytes for non-young regions.
@ -541,7 +544,8 @@ G1ParScanThreadState* G1ParScanThreadStateSet::state_for_worker(uint worker_id)
new G1ParScanThreadState(_g1h, rdcqs(),
_preserved_marks_set->get(worker_id),
worker_id, _n_workers,
_young_cset_length, _optional_cset_length);
_young_cset_length, _optional_cset_length,
_evac_failure_regions);
}
return _states[worker_id];
}
@ -595,7 +599,7 @@ oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, siz
// Forward-to-self succeeded. We are the "owner" of the object.
HeapRegion* r = _g1h->heap_region_containing(old);
if (_g1h->notify_region_failed_evacuation(r->hrm_index())) {
if (_evac_failure_regions->record(r->hrm_index())) {
_g1h->hr_printer()->evac_failure(r);
}
@ -649,7 +653,8 @@ G1ParScanThreadStateSet::G1ParScanThreadStateSet(G1CollectedHeap* g1h,
PreservedMarksSet* preserved_marks_set,
uint n_workers,
size_t young_cset_length,
size_t optional_cset_length) :
size_t optional_cset_length,
G1EvacFailureRegions* evac_failure_regions) :
_g1h(g1h),
_rdcqs(rdcqs),
_preserved_marks_set(preserved_marks_set),
@ -658,7 +663,8 @@ G1ParScanThreadStateSet::G1ParScanThreadStateSet(G1CollectedHeap* g1h,
_young_cset_length(young_cset_length),
_optional_cset_length(optional_cset_length),
_n_workers(n_workers),
_flushed(false) {
_flushed(false),
_evac_failure_regions(evac_failure_regions) {
for (uint i = 0; i < n_workers; ++i) {
_states[i] = NULL;
}

8
src/hotspot/share/gc/g1/g1ParScanThreadState.hpp
View File

@ -102,6 +102,7 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
// Per-thread evacuation failure data structures.
PreservedMarks* _preserved_marks;
EvacuationFailedInfo _evacuation_failed_info;
G1EvacFailureRegions* _evac_failure_regions;
void handle_evacuation_failure_notifications(oop obj, markWord m, size_t word_sz);
@ -112,7 +113,8 @@ public:
uint worker_id,
uint n_workers,
size_t young_cset_length,
size_t optional_cset_length);
size_t optional_cset_length,
G1EvacFailureRegions* evac_failure_regions);
virtual ~G1ParScanThreadState();
void set_ref_discoverer(ReferenceDiscoverer* rd) { _scanner.set_ref_discoverer(rd); }
@ -242,6 +244,7 @@ class G1ParScanThreadStateSet : public StackObj {
size_t _optional_cset_length;
uint _n_workers;
bool _flushed;
G1EvacFailureRegions* _evac_failure_regions;
public:
G1ParScanThreadStateSet(G1CollectedHeap* g1h,
@ -249,7 +252,8 @@ class G1ParScanThreadStateSet : public StackObj {
PreservedMarksSet* preserved_marks_set,
uint n_workers,
size_t young_cset_length,
size_t optional_cset_length);
size_t optional_cset_length,
G1EvacFailureRegions* evac_failure_regions);
~G1ParScanThreadStateSet();
G1RedirtyCardsQueueSet* rdcqs() { return _rdcqs; }

16
src/hotspot/share/gc/g1/g1YoungCollector.cpp
View File

@ -498,7 +498,7 @@ void G1YoungCollector::pre_evacuate_collection_set(G1EvacuationInfo* evacuation_
// reference processing currently works in G1.
ref_processor_stw()->start_discovery(false /* always_clear */);
_g1h->reset_evacuation_failed_data();
_evac_failure_regions->reset();
_g1h->gc_prologue(false);
@ -955,7 +955,7 @@ bool G1STWIsAliveClosure::do_object_b(oop p) {
void G1YoungCollector::post_evacuate_cleanup_1(G1ParScanThreadStateSet* per_thread_states) {
Ticks start = Ticks::now();
{
G1PostEvacuateCollectionSetCleanupTask1 cl(per_thread_states);
G1PostEvacuateCollectionSetCleanupTask1 cl(per_thread_states, _evac_failure_regions);
_g1h->run_batch_task(&cl);
}
phase_times()->record_post_evacuate_cleanup_task_1_time((Ticks::now() - start).seconds() * 1000.0);
@ -965,7 +965,7 @@ void G1YoungCollector::post_evacuate_cleanup_2(G1ParScanThreadStateSet* per_thre
G1EvacuationInfo* evacuation_info) {
Ticks start = Ticks::now();
{
G1PostEvacuateCollectionSetCleanupTask2 cl(per_thread_states, evacuation_info);
G1PostEvacuateCollectionSetCleanupTask2 cl(per_thread_states, evacuation_info, _evac_failure_regions);
_g1h->run_batch_task(&cl);
}
phase_times()->record_post_evacuate_cleanup_task_2_time((Ticks::now() - start).seconds() * 1000.0);
@ -1024,11 +1024,14 @@ public:
}
};
G1YoungCollector::G1YoungCollector(GCCause::Cause gc_cause, double target_pause_time_ms) :
G1YoungCollector::G1YoungCollector(GCCause::Cause gc_cause,
double target_pause_time_ms,
G1EvacFailureRegions* evac_failure_regions) :
_g1h(G1CollectedHeap::heap()),
_gc_cause(gc_cause),
_target_pause_time_ms(target_pause_time_ms),
_concurrent_operation_is_full_mark(false)
_concurrent_operation_is_full_mark(false),
_evac_failure_regions(evac_failure_regions)
{
}
@ -1080,7 +1083,8 @@ void G1YoungCollector::collect() {
&preserved_marks_set,
workers()->active_workers(),
collection_set()->young_region_length(),
collection_set()->optional_region_length());
collection_set()->optional_region_length(),
_evac_failure_regions);
pre_evacuate_collection_set(jtm.evacuation_info(), &per_thread_states);
bool may_do_optional_evacuation = collection_set()->optional_region_length() != 0;

6
src/hotspot/share/gc/g1/g1YoungCollector.hpp
View File

@ -37,6 +37,7 @@ class G1CollectedHeap;
class G1CollectionSet;
class G1CollectorState;
class G1ConcurrentMark;
class G1EvacFailureRegions;
class G1EvacuationInfo;
class G1GCPhaseTimes;
class G1HotCardCache;
@ -123,6 +124,7 @@ class G1YoungCollector {
void post_evacuate_collection_set(G1EvacuationInfo* evacuation_info,
G1ParScanThreadStateSet* per_thread_states);
G1EvacFailureRegions* _evac_failure_regions;
#if TASKQUEUE_STATS
uint num_task_queues() const;
@ -133,7 +135,9 @@ class G1YoungCollector {
public:
G1YoungCollector(GCCause::Cause gc_cause, double target_pause_time_ms);
G1YoungCollector(GCCause::Cause gc_cause,
double target_pause_time_ms,
G1EvacFailureRegions* evac_failure_regions);
void collect();
bool concurrent_operation_is_full_mark() const { return _concurrent_operation_is_full_mark; }

68
src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
View File

@ -39,7 +39,8 @@
#include "jfr/jfrEvents.hpp"
#include "utilities/ticks.hpp"
G1PostEvacuateCollectionSetCleanupTask1::G1PostEvacuateCollectionSetCleanupTask1(G1ParScanThreadStateSet* per_thread_states) :
G1PostEvacuateCollectionSetCleanupTask1::G1PostEvacuateCollectionSetCleanupTask1(G1ParScanThreadStateSet* per_thread_states,
G1EvacFailureRegions* evac_failure_regions) :
G1BatchedGangTask("Post Evacuate Cleanup 1", G1CollectedHeap::heap()->phase_times())
{
add_serial_task(new MergePssTask(per_thread_states));
@ -48,7 +49,7 @@ G1PostEvacuateCollectionSetCleanupTask1::G1PostEvacuateCollectionSetCleanupTask1
add_serial_task(new SampleCollectionSetCandidatesTask());
}
if (RemoveSelfForwardPtrsTask::should_execute()) {
add_parallel_task(new RemoveSelfForwardPtrsTask(per_thread_states->rdcqs()));
add_parallel_task(new RemoveSelfForwardPtrsTask(per_thread_states->rdcqs(), evac_failure_regions));
}
add_parallel_task(G1CollectedHeap::heap()->rem_set()->create_cleanup_after_scan_heap_roots_task());
}
@ -100,19 +101,23 @@ bool G1PostEvacuateCollectionSetCleanupTask1::RemoveSelfForwardPtrsTask::should_
return G1CollectedHeap::heap()->evacuation_failed();
}
G1PostEvacuateCollectionSetCleanupTask1::RemoveSelfForwardPtrsTask::RemoveSelfForwardPtrsTask(G1RedirtyCardsQueueSet* rdcqs) :
G1AbstractSubTask(G1GCPhaseTimes::RemoveSelfForwardingPtr), _task(rdcqs) { }
G1PostEvacuateCollectionSetCleanupTask1::
RemoveSelfForwardPtrsTask::RemoveSelfForwardPtrsTask(G1RedirtyCardsQueueSet* rdcqs,
G1EvacFailureRegions* evac_failure_regions) :
G1AbstractSubTask(G1GCPhaseTimes::RemoveSelfForwardingPtr),
_task(rdcqs, evac_failure_regions),
_evac_failure_regions(evac_failure_regions) { }
G1PostEvacuateCollectionSetCleanupTask1::RemoveSelfForwardPtrsTask::~RemoveSelfForwardPtrsTask() {
G1CollectedHeap* g1h = G1CollectedHeap::heap();
assert(_task.num_failed_regions() == g1h->num_regions_failed_evacuation(),
assert(_task.num_failed_regions() == _evac_failure_regions->num_regions_failed_evacuation(),
"Removed regions %u inconsistent with expected %u",
_task.num_failed_regions(), g1h->num_regions_failed_evacuation());
_task.num_failed_regions(), _evac_failure_regions->num_regions_failed_evacuation());
}
double G1PostEvacuateCollectionSetCleanupTask1::RemoveSelfForwardPtrsTask::worker_cost() const {
assert(should_execute(), "Should not call this if not executed");
return G1CollectedHeap::heap()->num_regions_failed_evacuation();
return _evac_failure_regions->num_regions_failed_evacuation();
}
void G1PostEvacuateCollectionSetCleanupTask1::RemoveSelfForwardPtrsTask::do_work(uint worker_id) {
@ -291,6 +296,7 @@ class RedirtyLoggedCardTableEntryClosure : public G1CardTableEntryClosure {
size_t _num_dirtied;
G1CollectedHeap* _g1h;
G1CardTable* _g1_ct;
G1EvacFailureRegions* _evac_failure_regions;
HeapRegion* region_for_card(CardValue* card_ptr) const {
return _g1h->heap_region_containing(_g1_ct->addr_for(card_ptr));
@ -299,12 +305,16 @@ class RedirtyLoggedCardTableEntryClosure : public G1CardTableEntryClosure {
bool will_become_free(HeapRegion* hr) const {
// A region will be freed by during the FreeCollectionSet phase if the region is in the
// collection set and has not had an evacuation failure.
return _g1h->is_in_cset(hr) && !_g1h->evacuation_failed(hr->hrm_index());
return _g1h->is_in_cset(hr) && !_evac_failure_regions->contains(hr->hrm_index());
}
public:
RedirtyLoggedCardTableEntryClosure(G1CollectedHeap* g1h) : G1CardTableEntryClosure(),
_num_dirtied(0), _g1h(g1h), _g1_ct(g1h->card_table()) { }
RedirtyLoggedCardTableEntryClosure(G1CollectedHeap* g1h, G1EvacFailureRegions* evac_failure_regions) :
G1CardTableEntryClosure(),
_num_dirtied(0),
_g1h(g1h),
_g1_ct(g1h->card_table()),
_evac_failure_regions(evac_failure_regions) { }
void do_card_ptr(CardValue* card_ptr, uint worker_id) {
HeapRegion* hr = region_for_card(card_ptr);
@ -319,10 +329,13 @@ class RedirtyLoggedCardTableEntryClosure : public G1CardTableEntryClosure {
size_t num_dirtied() const { return _num_dirtied; }
};
G1PostEvacuateCollectionSetCleanupTask2::RedirtyLoggedCardsTask::RedirtyLoggedCardsTask(G1RedirtyCardsQueueSet* rdcqs) :
G1PostEvacuateCollectionSetCleanupTask2::
RedirtyLoggedCardsTask::RedirtyLoggedCardsTask(G1RedirtyCardsQueueSet* rdcqs,
G1EvacFailureRegions* evac_failure_regions) :
G1AbstractSubTask(G1GCPhaseTimes::RedirtyCards),
_rdcqs(rdcqs),
_nodes(rdcqs->all_completed_buffers()) { }
_nodes(rdcqs->all_completed_buffers()),
_evac_failure_regions(evac_failure_regions) { }
G1PostEvacuateCollectionSetCleanupTask2::RedirtyLoggedCardsTask::~RedirtyLoggedCardsTask() {
G1DirtyCardQueueSet& dcq = G1BarrierSet::dirty_card_queue_set();
@ -336,7 +349,7 @@ double G1PostEvacuateCollectionSetCleanupTask2::RedirtyLoggedCardsTask::worker_c
}
void G1PostEvacuateCollectionSetCleanupTask2::RedirtyLoggedCardsTask::do_work(uint worker_id) {
RedirtyLoggedCardTableEntryClosure cl(G1CollectedHeap::heap());
RedirtyLoggedCardTableEntryClosure cl(G1CollectedHeap::heap(), _evac_failure_regions);
const size_t buffer_size = _rdcqs->buffer_size();
BufferNode* next = Atomic::load(&_nodes);
while (next != nullptr) {
@ -462,6 +475,7 @@ class FreeCSetClosure : public HeapRegionClosure {
Tickspan _young_time;
Tickspan _non_young_time;
FreeCSetStats* _stats;
G1EvacFailureRegions* _evac_failure_regions;
void assert_tracks_surviving_words(HeapRegion* r) {
assert(r->young_index_in_cset() != 0 &&
@ -504,14 +518,16 @@ class FreeCSetClosure : public HeapRegionClosure {
public:
FreeCSetClosure(const size_t* surviving_young_words,
uint worker_id,
FreeCSetStats* stats) :
FreeCSetStats* stats,
G1EvacFailureRegions* evac_failure_regions) :
HeapRegionClosure(),
_g1h(G1CollectedHeap::heap()),
_surviving_young_words(surviving_young_words),
_worker_id(worker_id),
_young_time(),
_non_young_time(),
_stats(stats) { }
_stats(stats),
_evac_failure_regions(evac_failure_regions) { }
virtual bool do_heap_region(HeapRegion* r) {
assert(r->in_collection_set(), "Invariant: %u missing from CSet", r->hrm_index());
@ -525,7 +541,7 @@ public:
r->record_surv_words_in_group(_surviving_young_words[r->young_index_in_cset()]);
}
if (_g1h->evacuation_failed(r->hrm_index())) {
if (_evac_failure_regions->contains(r->hrm_index())) {
handle_failed_region(r);
} else {
handle_evacuated_region(r);
@ -559,15 +575,18 @@ void G1PostEvacuateCollectionSetCleanupTask2::FreeCollectionSetTask::report_stat
total_stats.report(_g1h, _evacuation_info);
}
G1PostEvacuateCollectionSetCleanupTask2::FreeCollectionSetTask::FreeCollectionSetTask(G1EvacuationInfo* evacuation_info,
const size_t* surviving_young_words) :
G1PostEvacuateCollectionSetCleanupTask2::
FreeCollectionSetTask::FreeCollectionSetTask(G1EvacuationInfo* evacuation_info,
const size_t* surviving_young_words,
G1EvacFailureRegions* evac_failure_regions) :
G1AbstractSubTask(G1GCPhaseTimes::FreeCollectionSet),
_g1h(G1CollectedHeap::heap()),
_evacuation_info(evacuation_info),
_worker_stats(nullptr),
_claimer(0),
_surviving_young_words(surviving_young_words),
_active_workers(0) {
_active_workers(0),
_evac_failure_regions(evac_failure_regions) {
_g1h->clear_eden();
}
@ -596,14 +615,15 @@ void G1PostEvacuateCollectionSetCleanupTask2::FreeCollectionSetTask::set_max_wor
}
void G1PostEvacuateCollectionSetCleanupTask2::FreeCollectionSetTask::do_work(uint worker_id) {
FreeCSetClosure cl(_surviving_young_words, worker_id, worker_stats(worker_id));
FreeCSetClosure cl(_surviving_young_words, worker_id, worker_stats(worker_id), _evac_failure_regions);
_g1h->collection_set_par_iterate_all(&cl, &_claimer, worker_id);
// Report per-region type timings.
cl.report_timing();
}
G1PostEvacuateCollectionSetCleanupTask2::G1PostEvacuateCollectionSetCleanupTask2(G1ParScanThreadStateSet* per_thread_states,
G1EvacuationInfo* evacuation_info) :
G1EvacuationInfo* evacuation_info,
G1EvacFailureRegions* evac_failure_regions) :
G1BatchedGangTask("Post Evacuate Cleanup 2", G1CollectedHeap::heap()->phase_times())
{
add_serial_task(new ResetHotCardCacheTask());
@ -618,6 +638,8 @@ G1PostEvacuateCollectionSetCleanupTask2::G1PostEvacuateCollectionSetCleanupTask2
if (RestorePreservedMarksTask::should_execute()) {
add_parallel_task(new RestorePreservedMarksTask(per_thread_states->preserved_marks_set()));
}
add_parallel_task(new RedirtyLoggedCardsTask(per_thread_states->rdcqs()));
add_parallel_task(new FreeCollectionSetTask(evacuation_info, per_thread_states->surviving_young_words()));
add_parallel_task(new RedirtyLoggedCardsTask(per_thread_states->rdcqs(), evac_failure_regions));
add_parallel_task(new FreeCollectionSetTask(evacuation_info,
per_thread_states->surviving_young_words(),
evac_failure_regions));
}

18
src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp
View File

@ -31,6 +31,7 @@
class FreeCSetStats;
class G1CollectedHeap;
class G1EvacFailureRegions;
class G1EvacuationInfo;
class G1ParScanThreadStateSet;
class G1RedirtyCardsQueueSet;
@ -48,7 +49,8 @@ class G1PostEvacuateCollectionSetCleanupTask1 : public G1BatchedGangTask {
class RemoveSelfForwardPtrsTask;
public:
G1PostEvacuateCollectionSetCleanupTask1(G1ParScanThreadStateSet* per_thread_states);
G1PostEvacuateCollectionSetCleanupTask1(G1ParScanThreadStateSet* per_thread_states,
G1EvacFailureRegions* evac_failure_regions);
};
class G1PostEvacuateCollectionSetCleanupTask1::MergePssTask : public G1AbstractSubTask {
@ -81,9 +83,10 @@ public:
class G1PostEvacuateCollectionSetCleanupTask1::RemoveSelfForwardPtrsTask : public G1AbstractSubTask {
G1ParRemoveSelfForwardPtrsTask _task;
G1EvacFailureRegions* _evac_failure_regions;
public:
RemoveSelfForwardPtrsTask(G1RedirtyCardsQueueSet* rdcqs);
RemoveSelfForwardPtrsTask(G1RedirtyCardsQueueSet* rdcqs, G1EvacFailureRegions* evac_failure_regions);
~RemoveSelfForwardPtrsTask();
static bool should_execute();
@ -114,7 +117,8 @@ class G1PostEvacuateCollectionSetCleanupTask2 : public G1BatchedGangTask {
public:
G1PostEvacuateCollectionSetCleanupTask2(G1ParScanThreadStateSet* per_thread_states,
G1EvacuationInfo* evacuation_info);
G1EvacuationInfo* evacuation_info,
G1EvacFailureRegions* evac_failure_regions);
};
class G1PostEvacuateCollectionSetCleanupTask2::ResetHotCardCacheTask : public G1AbstractSubTask {
@ -174,9 +178,10 @@ public:
class G1PostEvacuateCollectionSetCleanupTask2::RedirtyLoggedCardsTask : public G1AbstractSubTask {
G1RedirtyCardsQueueSet* _rdcqs;
BufferNode* volatile _nodes;
G1EvacFailureRegions* _evac_failure_regions;
public:
RedirtyLoggedCardsTask(G1RedirtyCardsQueueSet* rdcqs);
RedirtyLoggedCardsTask(G1RedirtyCardsQueueSet* rdcqs, G1EvacFailureRegions* evac_failure_regions);
virtual ~RedirtyLoggedCardsTask();
double worker_cost() const override;
@ -190,12 +195,15 @@ class G1PostEvacuateCollectionSetCleanupTask2::FreeCollectionSetTask : public G1
HeapRegionClaimer _claimer;
const size_t* _surviving_young_words;
uint _active_workers;
G1EvacFailureRegions* _evac_failure_regions;
FreeCSetStats* worker_stats(uint worker);
void report_statistics();
public:
FreeCollectionSetTask(G1EvacuationInfo* evacuation_info, const size_t* surviving_young_words);
FreeCollectionSetTask(G1EvacuationInfo* evacuation_info,
const size_t* surviving_young_words,
G1EvacFailureRegions* evac_failure_regions);
virtual ~FreeCollectionSetTask();
double worker_cost() const override;