diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp index c880af7fd49..ca4dfc71c61 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp @@ -358,7 +358,7 @@ size_t ShenandoahGenerationalHeuristics::select_aged_regions(ShenandoahInPlacePr // Having chosen the collection set, adjust the budgets for generational mode based on its composition. Note // that young_generation->available() now knows about recently discovered immediate garbage. -void ShenandoahGenerationalHeuristics::adjust_evacuation_budgets(ShenandoahHeap* const heap, +void ShenandoahGenerationalHeuristics::adjust_evacuation_budgets(ShenandoahGenerationalHeap* const heap, ShenandoahCollectionSet* const collection_set) { shenandoah_assert_generational(); // We may find that old_evacuation_reserve and/or loaned_for_young_evacuation are not fully consumed, in which case we may @@ -481,6 +481,16 @@ void ShenandoahGenerationalHeuristics::adjust_evacuation_budgets(ShenandoahHeap* if (add_regions_to_young > 0) { assert(excess_old >= add_regions_to_young * region_size_bytes, "Cannot xfer more than excess old"); + if (heap->age_census()->is_always_tenure()) { + // Cap excess_old at one min-PLAB per worker so this much stays in old's promotion reserve + // instead of being transferred to young. + const size_t min_plab_total = heap->plab_min_size() * HeapWordSize * heap->workers()->max_workers(); + if (excess_old > min_plab_total) { + excess_old = min_plab_total; + // Avoid underflowing excess_old when we subtract below. + add_regions_to_young = 0; + } + } excess_old -= add_regions_to_young * region_size_bytes; log_debug(gc, ergo)("Before start of evacuation, total_promotion reserve is young_advance_promoted_reserve: %zu " "plus excess: old: %zu", young_advance_promoted_reserve_used, excess_old); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.hpp index a0e4ab78d5c..1860e3d4c0f 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.hpp @@ -92,7 +92,7 @@ private: // Adjust evacuation budgets after choosing collection set. On entry, the instance variable _regions_to_xfer // represents regions to be transferred to old based on decisions made in top_off_collection_set() - void adjust_evacuation_budgets(ShenandoahHeap* const heap, + void adjust_evacuation_budgets(ShenandoahGenerationalHeap* const heap, ShenandoahCollectionSet* const collection_set); protected: diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp index e4ac576aa6f..d9f3bdee828 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp @@ -179,6 +179,12 @@ void ShenandoahGlobalHeuristics::choose_global_collection_set(ShenandoahCollecti size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + original_young_evac_reserve; size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0; + // Admit every region with any garbage so every live object gets a chance to be promoted. + if (heap->age_census()->is_always_tenure()) { + ignore_threshold = 0; + min_garbage = SIZE_MAX; + } + ShenandoahGlobalCSetBudget budget(region_size_bytes, shared_reserve_regions * region_size_bytes, garbage_threshold, ignore_threshold, min_garbage, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp index 4989c929b32..8fa497802fd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp @@ -34,7 +34,7 @@ ShenandoahAgeCensus::ShenandoahAgeCensus() } ShenandoahAgeCensus::ShenandoahAgeCensus(uint max_workers) - : _max_workers(max_workers) + : _max_workers(max_workers), _always_tenure(false) { if (ShenandoahGenerationalMinTenuringAge > ShenandoahGenerationalMaxTenuringAge) { vm_exit_during_initialization( diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.hpp index c140f445e21..5636dee3ae2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.hpp @@ -121,6 +121,8 @@ class ShenandoahAgeCensus: public CHeapObj { uint _max_workers; // Maximum number of workers for parallel tasks + bool _always_tenure; // When true, every age is tenurable. + // Mortality rate of a cohort, given its population in // previous and current epochs double mortality_rate(size_t prev_pop, size_t cur_pop); @@ -150,9 +152,9 @@ class ShenandoahAgeCensus: public CHeapObj { return _tenuring_threshold[prev]; } - // Override the tenuring threshold for the current epoch. This is used to - // cause everything to be promoted for a whitebox full gc request. - void set_tenuring_threshold(uint threshold) { _tenuring_threshold[_epoch] = threshold; } + // Set always tenure mode. Currently only used by ShenandoahTenuringOverride + // to force is_tenurable() to be true for every age during WB.fullGC tests. + void set_always_tenure(bool always_tenure) { _always_tenure = always_tenure; } #ifndef PRODUCT // Return the sum of size of objects of all ages recorded in the @@ -187,11 +189,13 @@ class ShenandoahAgeCensus: public CHeapObj { // Visible for testing. Use is_tenurable for consistent tenuring comparisons. uint tenuring_threshold() const { return _tenuring_threshold[_epoch]; } - // Return true if this age is at or above the tenuring threshold. + // Return true if this age is at or above the tenuring threshold, or if always tenure is enabled. bool is_tenurable(uint age) const { - return age >= tenuring_threshold(); + return age >= tenuring_threshold() || _always_tenure; } + bool is_always_tenure() const { return _always_tenure; } + // Update the local age table for worker_id by size for // given obj_age, region_age, and region_youth CENSUS_NOISE(void add(uint obj_age, uint region_age, uint region_youth, size_t size, uint worker_id);) @@ -244,24 +248,22 @@ class ShenandoahAgeCensus: public CHeapObj { void print(); }; -// RAII object that temporarily overrides the tenuring threshold for the -// duration of a scope, restoring the original value on destruction. -// Used to force promotion of all young objects during whitebox full GCs. +// RAII object that enables ShenandoahAgeCensus always tenure mode for the +// duration of a scope and disables it on destruction. Used to force promotion +// of all young objects during whitebox full GCs. class ShenandoahTenuringOverride : public StackObj { ShenandoahAgeCensus* _census; - uint _saved_threshold; bool _active; public: ShenandoahTenuringOverride(bool active, ShenandoahAgeCensus* census) : - _census(census), _saved_threshold(0), _active(active) { + _census(census), _active(active) { if (_active) { - _saved_threshold = _census->tenuring_threshold(); - _census->set_tenuring_threshold(0); + _census->set_always_tenure(true); } } ~ShenandoahTenuringOverride() { if (_active) { - _census->set_tenuring_threshold(_saved_threshold); + _census->set_always_tenure(false); } } }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp index b3a48f85114..94f3409ac41 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp @@ -269,7 +269,7 @@ void ShenandoahGenerationalControlThread::run_gc_cycle(const ShenandoahGCRequest // Cannot uncommit bitmap slices during concurrent reset ShenandoahNoUncommitMark forbid_region_uncommit(_heap); - // When a whitebox full GC is requested, set the tenuring threshold to zero + // When a whitebox full GC is requested, set the age census to always tenure // so that all young objects are promoted to old. This ensures that tests // using WB.fullGC() to promote objects to old gen will not loop forever. ShenandoahTenuringOverride tenuring_override(request.cause == GCCause::_wb_full_gc, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPLAB.cpp b/src/hotspot/share/gc/shenandoah/shenandoahPLAB.cpp index 3d98d431b86..f139f94fc8b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPLAB.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPLAB.cpp @@ -39,7 +39,7 @@ ShenandoahPLAB::ShenandoahPLAB() : _promoted(0), _promotion_failure_count(0), _promotion_failure_words(0), - _allows_promotion(false), + _allows_promotion(true), _retries_enabled(false), _heap(ShenandoahGenerationalHeap::heap()) { _plab = new PLAB(PLAB::min_size());