diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp index 08fd4599346..dfae9040242 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp @@ -28,7 +28,7 @@ #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" #include "gc/shenandoah/shenandoahEvacInfo.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" -#include "gc/shenandoah/shenandoahGenerationalHeap.hpp" +#include "gc/shenandoah/shenandoahGenerationalHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahTrace.hpp" @@ -65,8 +65,6 @@ void ShenandoahGenerationalHeuristics::choose_collection_set(ShenandoahCollectio size_t free = 0; size_t free_regions = 0; - const uint tenuring_threshold = heap->age_census()->tenuring_threshold(); - // This counts number of humongous regions that we intend to promote in this cycle. size_t humongous_regions_promoted = 0; // This counts number of regular regions that will be promoted in place. @@ -98,12 +96,12 @@ void ShenandoahGenerationalHeuristics::choose_collection_set(ShenandoahCollectio bool is_candidate; // This is our candidate for later consideration. if (collection_set->is_preselected(i)) { - assert(region->age() >= tenuring_threshold, "Preselection filter"); + assert(heap->is_tenurable(region), "Preselection filter"); is_candidate = true; preselected_candidates++; // Set garbage value to maximum value to force this into the sorted collection set. garbage = region_size_bytes; - } else if (region->is_young() && (region->age() >= tenuring_threshold)) { + } else if (region->is_young() && heap->is_tenurable(region)) { // Note that for GLOBAL GC, region may be OLD, and OLD regions do not qualify for pre-selection // This region is old enough to be promoted but it was not preselected, either because its garbage is below @@ -142,7 +140,7 @@ void ShenandoahGenerationalHeuristics::choose_collection_set(ShenandoahCollectio immediate_regions++; immediate_garbage += garbage; } else { - if (region->is_young() && region->age() >= tenuring_threshold) { + if (region->is_young() && heap->is_tenurable(region)) { oop obj = cast_to_oop(region->bottom()); size_t humongous_regions = ShenandoahHeapRegion::required_regions(obj->size() * HeapWordSize); humongous_regions_promoted += humongous_regions; @@ -246,10 +244,6 @@ void ShenandoahGenerationalHeuristics::choose_collection_set(ShenandoahCollectio size_t ShenandoahGenerationalHeuristics::add_preselected_regions_to_collection_set(ShenandoahCollectionSet* cset, const RegionData* data, size_t size) const { -#ifdef ASSERT - const uint tenuring_threshold = ShenandoahGenerationalHeap::heap()->age_census()->tenuring_threshold(); -#endif - // cur_young_garbage represents the amount of memory to be reclaimed from young-gen. In the case that live objects // are known to be promoted out of young-gen, we count this as cur_young_garbage because this memory is reclaimed // from young-gen and becomes available to serve future young-gen allocation requests. @@ -257,7 +251,7 @@ size_t ShenandoahGenerationalHeuristics::add_preselected_regions_to_collection_s for (size_t idx = 0; idx < size; idx++) { ShenandoahHeapRegion* r = data[idx].get_region(); if (cset->is_preselected(r->index())) { - assert(r->age() >= tenuring_threshold, "Preselected regions must have tenure age"); + assert(ShenandoahGenerationalHeap::heap()->is_tenurable(r), "Preselected regions must have tenure age"); // Entire region will be promoted, This region does not impact young-gen or old-gen evacuation reserve. // This region has been pre-selected and its impact on promotion reserve is already accounted for. diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp index 4e12b1d41e8..331bd040575 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp @@ -25,7 +25,7 @@ #include "gc/shenandoah/heuristics/shenandoahGlobalHeuristics.hpp" #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" -#include "gc/shenandoah/shenandoahGenerationalHeap.hpp" +#include "gc/shenandoah/shenandoahGenerationalHeap.inline.hpp" #include "gc/shenandoah/shenandoahGlobalGeneration.hpp" #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "utilities/quickSort.hpp" @@ -56,7 +56,6 @@ void ShenandoahGlobalHeuristics::choose_global_collection_set(ShenandoahCollecti size_t capacity = heap->soft_max_capacity(); size_t garbage_threshold = region_size_bytes * ShenandoahGarbageThreshold / 100; size_t ignore_threshold = region_size_bytes * ShenandoahIgnoreGarbageThreshold / 100; - const uint tenuring_threshold = heap->age_census()->tenuring_threshold(); size_t young_evac_reserve = heap->young_generation()->get_evacuation_reserve(); size_t old_evac_reserve = heap->old_generation()->get_evacuation_reserve(); @@ -100,7 +99,7 @@ void ShenandoahGlobalHeuristics::choose_global_collection_set(ShenandoahCollecti ShenandoahHeapRegion* r = data[idx].get_region(); assert(!cset->is_preselected(r->index()), "There should be no preselected regions during GLOBAL GC"); bool add_region = false; - if (r->is_old() || (r->age() >= tenuring_threshold)) { + if (r->is_old() || heap->is_tenurable(r)) { size_t new_cset = old_cur_cset + r->get_live_data_bytes(); if ((r->garbage() > garbage_threshold)) { while ((new_cset > max_old_cset) && (unaffiliated_young_regions > 0)) { @@ -114,7 +113,7 @@ void ShenandoahGlobalHeuristics::choose_global_collection_set(ShenandoahCollecti old_cur_cset = new_cset; } } else { - assert(r->is_young() && (r->age() < tenuring_threshold), "DeMorgan's law (assuming r->is_affiliated)"); + assert(r->is_young() && !heap->is_tenurable(r), "DeMorgan's law (assuming r->is_affiliated)"); size_t new_cset = young_cur_cset + r->get_live_data_bytes(); size_t region_garbage = r->garbage(); size_t new_garbage = cur_young_garbage + region_garbage; diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.cpp index fbb165858dc..d236be8c9e6 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.cpp @@ -26,7 +26,7 @@ #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp" #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" -#include "gc/shenandoah/shenandoahGenerationalHeap.hpp" +#include "gc/shenandoah/shenandoahGenerationalHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" @@ -64,19 +64,18 @@ void ShenandoahYoungHeuristics::choose_young_collection_set(ShenandoahCollection size_t size, size_t actual_free, size_t cur_young_garbage) const { - auto heap = ShenandoahGenerationalHeap::heap(); + const auto heap = ShenandoahGenerationalHeap::heap(); - size_t capacity = heap->soft_max_capacity(); - size_t garbage_threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahGarbageThreshold / 100; - size_t ignore_threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahIgnoreGarbageThreshold / 100; - const uint tenuring_threshold = heap->age_census()->tenuring_threshold(); + const size_t capacity = heap->soft_max_capacity(); + const size_t garbage_threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahGarbageThreshold / 100; + const size_t ignore_threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahIgnoreGarbageThreshold / 100; // This is young-gen collection or a mixed evacuation. // If this is mixed evacuation, the old-gen candidate regions have already been added. - size_t max_cset = (size_t) (heap->young_generation()->get_evacuation_reserve() / ShenandoahEvacWaste); size_t cur_cset = 0; - size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + max_cset; - size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0; + const size_t max_cset = (size_t) (heap->young_generation()->get_evacuation_reserve() / ShenandoahEvacWaste); + const size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + max_cset; + const size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0; log_info(gc, ergo)( @@ -89,11 +88,15 @@ void ShenandoahYoungHeuristics::choose_young_collection_set(ShenandoahCollection if (cset->is_preselected(r->index())) { continue; } - if (r->age() < tenuring_threshold) { - size_t new_cset = cur_cset + r->get_live_data_bytes(); - size_t region_garbage = r->garbage(); - size_t new_garbage = cur_young_garbage + region_garbage; - bool add_regardless = (region_garbage > ignore_threshold) && (new_garbage < min_garbage); + + // Note that we do not add tenurable regions if they were not pre-selected. They were not preselected + // because there is insufficient room in old-gen to hold their to-be-promoted live objects or because + // they are to be promoted in place. + if (!heap->is_tenurable(r)) { + const size_t new_cset = cur_cset + r->get_live_data_bytes(); + const size_t region_garbage = r->garbage(); + const size_t new_garbage = cur_young_garbage + region_garbage; + const bool add_regardless = (region_garbage > ignore_threshold) && (new_garbage < min_garbage); assert(r->is_young(), "Only young candidates expected in the data array"); if ((new_cset <= max_cset) && (add_regardless || (region_garbage > garbage_threshold))) { cur_cset = new_cset; @@ -101,9 +104,6 @@ void ShenandoahYoungHeuristics::choose_young_collection_set(ShenandoahCollection cset->add_region(r); } } - // Note that we do not add aged regions if they were not pre-selected. The reason they were not preselected - // is because there is not sufficient room in old-gen to hold their to-be-promoted live objects or because - // they are to be promoted in place. } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp index 94c98b78f1b..bd66f55bd8f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp @@ -27,8 +27,15 @@ #include "gc/shenandoah/shenandoahAgeCensus.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" -ShenandoahAgeCensus::ShenandoahAgeCensus() { +ShenandoahAgeCensus::ShenandoahAgeCensus() + : ShenandoahAgeCensus(ShenandoahHeap::heap()->max_workers()) +{ assert(ShenandoahHeap::heap()->mode()->is_generational(), "Only in generational mode"); +} + +ShenandoahAgeCensus::ShenandoahAgeCensus(uint max_workers) + : _max_workers(max_workers) +{ if (ShenandoahGenerationalMinTenuringAge > ShenandoahGenerationalMaxTenuringAge) { vm_exit_during_initialization( err_msg("ShenandoahGenerationalMinTenuringAge=%zu" @@ -39,6 +46,9 @@ ShenandoahAgeCensus::ShenandoahAgeCensus() { _global_age_table = NEW_C_HEAP_ARRAY(AgeTable*, MAX_SNAPSHOTS, mtGC); CENSUS_NOISE(_global_noise = NEW_C_HEAP_ARRAY(ShenandoahNoiseStats, MAX_SNAPSHOTS, mtGC);) _tenuring_threshold = NEW_C_HEAP_ARRAY(uint, MAX_SNAPSHOTS, mtGC); + CENSUS_NOISE(_skipped = 0); + NOT_PRODUCT(_counted = 0); + NOT_PRODUCT(_total = 0); for (int i = 0; i < MAX_SNAPSHOTS; i++) { // Note that we don't now get perfdata from age_table @@ -48,10 +58,9 @@ ShenandoahAgeCensus::ShenandoahAgeCensus() { _tenuring_threshold[i] = MAX_COHORTS; } if (ShenandoahGenerationalAdaptiveTenuring && !ShenandoahGenerationalCensusAtEvac) { - size_t max_workers = ShenandoahHeap::heap()->max_workers(); - _local_age_table = NEW_C_HEAP_ARRAY(AgeTable*, max_workers, mtGC); + _local_age_table = NEW_C_HEAP_ARRAY(AgeTable*, _max_workers, mtGC); CENSUS_NOISE(_local_noise = NEW_C_HEAP_ARRAY(ShenandoahNoiseStats, max_workers, mtGC);) - for (uint i = 0; i < max_workers; i++) { + for (uint i = 0; i < _max_workers; i++) { _local_age_table[i] = new AgeTable(false); CENSUS_NOISE(_local_noise[i].clear();) } @@ -61,6 +70,22 @@ ShenandoahAgeCensus::ShenandoahAgeCensus() { _epoch = MAX_SNAPSHOTS - 1; // see update_epoch() } +ShenandoahAgeCensus::~ShenandoahAgeCensus() { + for (uint i = 0; i < MAX_SNAPSHOTS; i++) { + delete _global_age_table[i]; + } + FREE_C_HEAP_ARRAY(AgeTable*, _global_age_table); + FREE_C_HEAP_ARRAY(uint, _tenuring_threshold); + CENSUS_NOISE(FREE_C_HEAP_ARRAY(ShenandoahNoiseStats, _global_noise)); + if (_local_age_table) { + for (uint i = 0; i < _max_workers; i++) { + delete _local_age_table[i]; + } + FREE_C_HEAP_ARRAY(AgeTable*, _local_age_table); + CENSUS_NOISE(FREE_C_HEAP_ARRAY(ShenandoahNoiseStats, _local_noise)); + } +} + CENSUS_NOISE(void ShenandoahAgeCensus::add(uint obj_age, uint region_age, uint region_youth, size_t size, uint worker_id) {) NO_CENSUS_NOISE(void ShenandoahAgeCensus::add(uint obj_age, uint region_age, size_t size, uint worker_id) {) if (obj_age <= markWord::max_age) { @@ -131,12 +156,11 @@ void ShenandoahAgeCensus::update_census(size_t age0_pop, AgeTable* pv1, AgeTable assert(pv1 == nullptr && pv2 == nullptr, "Error, check caller"); // Seed cohort 0 with population that may have been missed during // regular census. - _global_age_table[_epoch]->add((uint)0, age0_pop); + _global_age_table[_epoch]->add(0u, age0_pop); - size_t max_workers = ShenandoahHeap::heap()->max_workers(); // Merge data from local age tables into the global age table for the epoch, // clearing the local tables. - for (uint i = 0; i < max_workers; i++) { + for (uint i = 0; i < _max_workers; i++) { // age stats _global_age_table[_epoch]->merge(_local_age_table[i]); _local_age_table[i]->clear(); // clear for next census @@ -177,8 +201,7 @@ void ShenandoahAgeCensus::reset_local() { assert(_local_age_table == nullptr, "Error"); return; } - size_t max_workers = ShenandoahHeap::heap()->max_workers(); - for (uint i = 0; i < max_workers; i++) { + for (uint i = 0; i < _max_workers; i++) { _local_age_table[i]->clear(); CENSUS_NOISE(_local_noise[i].clear();) } @@ -204,8 +227,7 @@ bool ShenandoahAgeCensus::is_clear_local() { assert(_local_age_table == nullptr, "Error"); return true; } - size_t max_workers = ShenandoahHeap::heap()->max_workers(); - for (uint i = 0; i < max_workers; i++) { + for (uint i = 0; i < _max_workers; i++) { bool clear = _local_age_table[i]->is_clear(); CENSUS_NOISE(clear |= _local_noise[i].is_clear();) if (!clear) { @@ -246,7 +268,7 @@ void ShenandoahAgeCensus::update_tenuring_threshold() { _tenuring_threshold[_epoch] = tt; } print(); - log_trace(gc, age)("New tenuring threshold %zu (min %zu, max %zu)", + log_info(gc, age)("New tenuring threshold %zu (min %zu, max %zu)", (uintx) _tenuring_threshold[_epoch], ShenandoahGenerationalMinTenuringAge, ShenandoahGenerationalMaxTenuringAge); } @@ -279,13 +301,14 @@ uint ShenandoahAgeCensus::compute_tenuring_threshold() { uint upper_bound = ShenandoahGenerationalMaxTenuringAge; const uint prev_tt = previous_tenuring_threshold(); if (ShenandoahGenerationalCensusIgnoreOlderCohorts && prev_tt > 0) { - // We stay below the computed tenuring threshold for the last cycle plus 1, - // ignoring the mortality rates of any older cohorts. - upper_bound = MIN2(upper_bound, prev_tt + 1); + // We stay below the computed tenuring threshold for the last cycle, + // ignoring the mortality rates of any older cohorts (which may see + // higher mortality rates due to promotions). + upper_bound = MIN2(upper_bound, prev_tt); } upper_bound = MIN2(upper_bound, markWord::max_age); - const uint lower_bound = MAX2((uint)ShenandoahGenerationalMinTenuringAge, (uint)1); + const uint lower_bound = MAX2((uint)ShenandoahGenerationalMinTenuringAge, 1u); uint tenuring_threshold = upper_bound; for (uint i = upper_bound; i >= lower_bound; i--) { @@ -303,9 +326,9 @@ uint ShenandoahAgeCensus::compute_tenuring_threshold() { // cohorts are considered eligible for tenuring when all older // cohorts are. We return the next higher age as the tenuring threshold // so that we do not prematurely promote objects of this age. - assert(tenuring_threshold == i+1 || tenuring_threshold == upper_bound, "Error"); + assert(tenuring_threshold == i + 1 || tenuring_threshold == upper_bound, "Error"); assert(tenuring_threshold >= lower_bound && tenuring_threshold <= upper_bound, "Error"); - return tenuring_threshold; + return i + 1; } // Remember that we passed over this cohort, looking for younger cohorts // showing high mortality. We want to tenure cohorts of this age. @@ -335,6 +358,14 @@ double ShenandoahAgeCensus::mortality_rate(size_t prev_pop, size_t cur_pop) { } void ShenandoahAgeCensus::print() { + + const LogTarget(Debug, gc, age) lt; + if (!lt.is_enabled()) { + return; + } + + LogStream ls(lt); + // Print the population vector for the current epoch, and // for the previous epoch, as well as the computed mortality // ratio for each extant cohort. @@ -350,33 +381,32 @@ void ShenandoahAgeCensus::print() { for (uint i = 1; i < MAX_COHORTS; i++) { const size_t prev_pop = prev_pv->sizes[i-1]; // (i-1) OK because i >= 1 const size_t cur_pop = cur_pv->sizes[i]; - double mr = mortality_rate(prev_pop, cur_pop); + const double mr = mortality_rate(prev_pop, cur_pop); // Suppress printing when everything is zero if (prev_pop + cur_pop > 0) { - log_info(gc, age) - (" - age %3u: prev %10zu bytes, curr %10zu bytes, mortality %.2f ", - i, prev_pop*oopSize, cur_pop*oopSize, mr); + ls.print_cr(" - age %3u: prev %10zu bytes, curr %10zu bytes, mortality %.2f ", + i, prev_pop * oopSize, cur_pop * oopSize, mr); } total += cur_pop; if (i == tt) { // Underline the cohort for tenuring threshold (if < MAX_COHORTS) - log_info(gc, age)("----------------------------------------------------------------------------"); + ls.print_cr("----------------------------------------------------------------------------"); } } - CENSUS_NOISE(_global_noise[cur_epoch].print(total);) + CENSUS_NOISE(_global_noise[cur_epoch].print(ls, total);) } #ifdef SHENANDOAH_CENSUS_NOISE -void ShenandoahNoiseStats::print(size_t total) { +void ShenandoahNoiseStats::print(LogStream& ls, const size_t total) { if (total > 0) { - float f_skipped = (float)skipped/(float)total; - float f_aged = (float)aged/(float)total; - float f_clamped = (float)clamped/(float)total; - float f_young = (float)young/(float)total; - log_info(gc, age)("Skipped: %10zu (%.2f), R-Aged: %10zu (%.2f), " - "Clamped: %10zu (%.2f), R-Young: %10zu (%.2f)", - skipped*oopSize, f_skipped, aged*oopSize, f_aged, - clamped*oopSize, f_clamped, young*oopSize, f_young); + const float f_skipped = (float)skipped/(float)total; + const float f_aged = (float)aged/(float)total; + const float f_clamped = (float)clamped/(float)total; + const float f_young = (float)young/(float)total; + ls.print_cr("Skipped: %10zu (%.2f), R-Aged: %10zu (%.2f), " + "Clamped: %10zu (%.2f), R-Young: %10zu (%.2f)", + skipped*oopSize, f_skipped, aged*oopSize, f_aged, + clamped*oopSize, f_clamped, young*oopSize, f_young); } } #endif // SHENANDOAH_CENSUS_NOISE diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.hpp index 89c68f7120b..90d188e1fca 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.hpp @@ -37,6 +37,8 @@ #define CENSUS_NOISE(x) x #define NO_CENSUS_NOISE(x) +class LogStream; + struct ShenandoahNoiseStats { size_t skipped; // Volume of objects skipped size_t aged; // Volume of objects from aged regions @@ -67,7 +69,7 @@ struct ShenandoahNoiseStats { young += other.young; } - void print(size_t total); + void print(LogStream& ls, size_t total); }; #else // SHENANDOAH_CENSUS_NOISE #define CENSUS_NOISE(x) @@ -91,7 +93,7 @@ struct ShenandoahNoiseStats { // // In addition, this class also maintains per worker population vectors into which // census for the current minor GC is accumulated (during marking or, optionally, during -// evacuation). These are cleared after each marking (resectively, evacuation) cycle, +// evacuation). These are cleared after each marking (respectively, evacuation) cycle, // once the per-worker data is consolidated into the appropriate population vector // per minor collection. The _local_age_table is thus C x N, for N GC workers. class ShenandoahAgeCensus: public CHeapObj { @@ -111,10 +113,12 @@ class ShenandoahAgeCensus: public CHeapObj { size_t _total; // net size of objects encountered (counted or skipped) in census #endif - uint _epoch; // Current epoch (modulo max age) - uint *_tenuring_threshold; // An array of the last N tenuring threshold values we + uint _epoch; // Current epoch (modulo max age) + uint* _tenuring_threshold; // An array of the last N tenuring threshold values we // computed. + uint _max_workers; // Maximum number of workers for parallel tasks + // Mortality rate of a cohort, given its population in // previous and current epochs double mortality_rate(size_t prev_pop, size_t cur_pop); @@ -165,11 +169,22 @@ class ShenandoahAgeCensus: public CHeapObj { }; ShenandoahAgeCensus(); + ShenandoahAgeCensus(uint max_workers); + ~ShenandoahAgeCensus(); // Return the local age table (population vector) for worker_id. // Only used in the case of (ShenandoahGenerationalAdaptiveTenuring && !ShenandoahGenerationalCensusAtEvac) - AgeTable* get_local_age_table(uint worker_id) { - return (AgeTable*) _local_age_table[worker_id]; + AgeTable* get_local_age_table(uint worker_id) const { + return _local_age_table[worker_id]; + } + + // Return the most recently computed tenuring threshold. + // Visible for testing. Use is_tenurable for consistent tenuring comparisons. + uint tenuring_threshold() const { return _tenuring_threshold[_epoch]; } + + // Return true if this age is at or above the tenuring threshold. + bool is_tenurable(uint age) const { + return age >= tenuring_threshold(); } // Update the local age table for worker_id by size for @@ -201,9 +216,6 @@ class ShenandoahAgeCensus: public CHeapObj { // is 0, because the evacuated objects have all had their ages incremented. void update_census(size_t age0_pop, AgeTable* pv1 = nullptr, AgeTable* pv2 = nullptr); - // Return the most recently computed tenuring threshold - uint tenuring_threshold() const { return _tenuring_threshold[_epoch]; } - // Reset the epoch, clearing accumulated census history // Note: this isn't currently used, but reserved for planned // future usage. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp index 25b900f8d77..35faa40af77 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp @@ -27,6 +27,7 @@ #include "gc/shenandoah/shenandoahAgeCensus.hpp" #include "gc/shenandoah/shenandoahCollectionSet.hpp" +#include "gc/shenandoah/shenandoahGenerationalHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegionSet.hpp" @@ -98,7 +99,7 @@ void ShenandoahCollectionSet::add_region(ShenandoahHeapRegion* r) { if (r->is_young()) { _young_bytes_to_evacuate += live; _young_available_bytes_collected += free; - if (ShenandoahHeap::heap()->mode()->is_generational() && r->age() >= ShenandoahGenerationalHeap::heap()->age_census()->tenuring_threshold()) { + if (ShenandoahHeap::heap()->mode()->is_generational() && ShenandoahGenerationalHeap::heap()->is_tenurable(r)) { _young_bytes_to_promote += live; } } else if (r->is_old()) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index f686334d3d5..7b3839dc198 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -28,9 +28,8 @@ #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" -#include "gc/shenandoah/shenandoahGenerationalHeap.hpp" +#include "gc/shenandoah/shenandoahGenerationalHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp" -#include "gc/shenandoah/shenandoahMonitoringSupport.hpp" #include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" @@ -534,7 +533,6 @@ size_t ShenandoahGeneration::select_aged_regions(size_t old_available) { bool* const candidate_regions_for_promotion_by_copy = heap->collection_set()->preselected_regions(); ShenandoahMarkingContext* const ctx = heap->marking_context(); - const uint tenuring_threshold = heap->age_census()->tenuring_threshold(); const size_t old_garbage_threshold = (ShenandoahHeapRegion::region_size_bytes() * ShenandoahOldGarbageThreshold) / 100; size_t old_consumed = 0; @@ -558,7 +556,7 @@ size_t ShenandoahGeneration::select_aged_regions(size_t old_available) { // skip over regions that aren't regular young with some live data continue; } - if (r->age() >= tenuring_threshold) { + if (heap->is_tenurable(r)) { if ((r->garbage() < old_garbage_threshold)) { // This tenure-worthy region has too little garbage, so we do not want to expend the copying effort to // reclaim the garbage; instead this region may be eligible for promotion-in-place to the @@ -613,7 +611,7 @@ size_t ShenandoahGeneration::select_aged_regions(size_t old_available) { // these regions. The likely outcome is that these regions will not be selected for evacuation or promotion // in the current cycle and we will anticipate that they will be promoted in the next cycle. This will cause // us to reserve more old-gen memory so that these objects can be promoted in the subsequent cycle. - if (heap->is_aging_cycle() && (r->age() + 1 == tenuring_threshold)) { + if (heap->is_aging_cycle() && heap->age_census()->is_tenurable(r->age() + 1)) { if (r->garbage() >= old_garbage_threshold) { promo_potential += r->get_live_data_bytes(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp index 29fd3258b6c..3a0d7926865 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp @@ -26,7 +26,7 @@ #include "gc/shenandoah/shenandoahAsserts.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp" -#include "gc/shenandoah/shenandoahGenerationalHeap.hpp" +#include "gc/shenandoah/shenandoahGenerationalHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" @@ -56,11 +56,9 @@ ShenandoahGenerationalEvacuationTask::ShenandoahGenerationalEvacuationTask(Shena _heap(heap), _regions(iterator), _concurrent(concurrent), - _only_promote_regions(only_promote_regions), - _tenuring_threshold(0) + _only_promote_regions(only_promote_regions) { shenandoah_assert_generational(); - _tenuring_threshold = _heap->age_census()->tenuring_threshold(); } void ShenandoahGenerationalEvacuationTask::work(uint worker_id) { @@ -138,7 +136,7 @@ void ShenandoahGenerationalEvacuationTask::evacuate_and_promote_regions() { void ShenandoahGenerationalEvacuationTask::maybe_promote_region(ShenandoahHeapRegion* r) { - if (r->is_young() && r->is_active() && (r->age() >= _tenuring_threshold)) { + if (r->is_young() && r->is_active() && _heap->is_tenurable(r)) { if (r->is_humongous_start()) { // We promote humongous_start regions along with their affiliated continuations during evacuation rather than // doing this work during a safepoint. We cannot put humongous regions into the collection set because that @@ -176,7 +174,7 @@ void ShenandoahGenerationalEvacuationTask::promote_in_place(ShenandoahHeapRegion assert(region->garbage_before_padded_for_promote() < old_garbage_threshold, "Region %zu has too much garbage for promotion", region->index()); assert(region->is_young(), "Only young regions can be promoted"); assert(region->is_regular(), "Use different service to promote humongous regions"); - assert(region->age() >= _tenuring_threshold, "Only promote regions that are sufficiently aged"); + assert(_heap->is_tenurable(region), "Only promote regions that are sufficiently aged"); assert(region->get_top_before_promote() == tams, "Region %zu has been used for allocations before promotion", region->index()); } @@ -259,7 +257,7 @@ void ShenandoahGenerationalEvacuationTask::promote_humongous(ShenandoahHeapRegio shenandoah_assert_generations_reconciled(); assert(region->is_young(), "Only young regions can be promoted"); assert(region->is_humongous_start(), "Should not promote humongous continuation in isolation"); - assert(region->age() >= _tenuring_threshold, "Only promote regions that are sufficiently aged"); + assert(_heap->is_tenurable(region), "Only promote regions that are sufficiently aged"); assert(marking_context->is_marked(obj), "promoted humongous object should be alive"); const size_t used_bytes = obj->size() * HeapWordSize; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp index abe2fc0110c..0c402d6c90a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp @@ -39,7 +39,6 @@ private: ShenandoahRegionIterator* _regions; bool _concurrent; bool _only_promote_regions; - uint _tenuring_threshold; public: ShenandoahGenerationalEvacuationTask(ShenandoahGenerationalHeap* sh, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.cpp index e2e3f0a4677..c4a7408e032 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.cpp @@ -193,7 +193,6 @@ ShenandoahPrepareForGenerationalCompactionObjectClosure::ShenandoahPrepareForGen ShenandoahHeapRegion* from_region, uint worker_id) : _preserved_marks(preserved_marks), _heap(ShenandoahGenerationalHeap::heap()), - _tenuring_threshold(0), _empty_regions(empty_regions), _empty_regions_pos(0), _old_to_region(nullptr), @@ -212,8 +211,6 @@ ShenandoahPrepareForGenerationalCompactionObjectClosure::ShenandoahPrepareForGen _young_to_region = from_region; _young_compact_point = from_region->bottom(); } - - _tenuring_threshold = _heap->age_census()->tenuring_threshold(); } void ShenandoahPrepareForGenerationalCompactionObjectClosure::set_from_region(ShenandoahHeapRegion* from_region) { @@ -279,7 +276,7 @@ void ShenandoahPrepareForGenerationalCompactionObjectClosure::do_object(oop p) { bool promote_object = false; if ((_from_affiliation == ShenandoahAffiliation::YOUNG_GENERATION) && - (from_region_age + object_age >= _tenuring_threshold)) { + _heap->age_census()->is_tenurable(from_region_age + object_age)) { if ((_old_to_region != nullptr) && (_old_compact_point + obj_size > _old_to_region->end())) { finish_old_region(); _old_to_region = nullptr; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.hpp index 9240a056105..06080286f22 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.hpp @@ -90,7 +90,6 @@ class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClo private: PreservedMarks* const _preserved_marks; ShenandoahGenerationalHeap* const _heap; - uint _tenuring_threshold; // _empty_regions is a thread-local list of heap regions that have been completely emptied by this worker thread's // compaction efforts. The worker thread that drives these efforts adds compacted regions to this list if the diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp index d05ae713645..0aca8f971e3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp @@ -216,7 +216,7 @@ oop ShenandoahGenerationalHeap::evacuate_object(oop p, Thread* thread) { if (mark.has_displaced_mark_helper()) { // We don't want to deal with MT here just to ensure we read the right mark word. // Skip the potential promotion attempt for this one. - } else if (r->age() + mark.age() >= age_census()->tenuring_threshold()) { + } else if (age_census()->is_tenurable(r->age() + mark.age())) { oop result = try_evacuate_object(p, thread, r, OLD_GENERATION); if (result != nullptr) { return result; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp index f23e49735e9..6960562b31d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp @@ -80,6 +80,7 @@ public: return _age_census; } + inline bool is_tenurable(const ShenandoahHeapRegion* r) const; // Ages regions that haven't been used for allocations in the current cycle. // Resets ages for regions that have been used for allocations. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.inline.hpp new file mode 100644 index 00000000000..8289b48185b --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.inline.hpp @@ -0,0 +1,37 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONALHEAP_INLINE_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONALHEAP_INLINE_HPP + +#include "gc/shenandoah/shenandoahGenerationalHeap.hpp" + +#include "gc/shenandoah/shenandoahAgeCensus.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" + +inline bool ShenandoahGenerationalHeap::is_tenurable(const ShenandoahHeapRegion* r) const { + return _age_census->is_tenurable(r->age()); +} + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONALHEAP_INLINE_HPP diff --git a/test/hotspot/gtest/gc/shenandoah/test_shenandoahAgeCensus.cpp b/test/hotspot/gtest/gc/shenandoah/test_shenandoahAgeCensus.cpp new file mode 100644 index 00000000000..c53d0a15554 --- /dev/null +++ b/test/hotspot/gtest/gc/shenandoah/test_shenandoahAgeCensus.cpp @@ -0,0 +1,159 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "gc/shenandoah/shenandoahAgeCensus.hpp" +#include "unittest.hpp" + +class ShenandoahAgeCensusTest : public ::testing::Test { +protected: + static constexpr size_t MinimumPopulationSize = 4*K; + static constexpr size_t InitialPopulationSize = MinimumPopulationSize * 1000; + + size_t _cohorts_count = ShenandoahAgeCensus::MAX_COHORTS; + double _mortality_rates[ShenandoahAgeCensus::MAX_COHORTS]; + size_t _cohort_populations[ShenandoahAgeCensus::MAX_COHORTS]; + + ShenandoahAgeCensusTest() + : _mortality_rates{0.9, 0.7, 0.5, 0.3, 0.09, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0} + { + build_cohort_populations(_mortality_rates, _cohort_populations, _cohorts_count); + } + + static void add_population(ShenandoahAgeCensus& census, const uint age, const size_t population_words) { + CENSUS_NOISE(census.add(age, 0, 0, population_words, 0)); + NO_CENSUS_NOISE(census.add(age, 0, population_words, 0)); + } + + void update(ShenandoahAgeCensus& census, size_t cohorts) const { + for (uint i = 1; i < cohorts; i++) { + add_population(census, i, _cohort_populations[i]); + } + census.update_census(_cohort_populations[0]); + } + + void update(ShenandoahAgeCensus& census) const { + update(census, _cohorts_count); + } + + size_t get_total_population_older_than(const size_t min_cohort_age) const { + size_t total = 0; + for (size_t i = 0; i < _cohorts_count; i++) { + if (i >= min_cohort_age) { + total += _cohort_populations[i]; + } + } + return total; + } + + void promote_all_tenurable(const size_t tenuring_threshold) { + for (size_t i = 0; i < _cohorts_count; i++) { + if (i > tenuring_threshold) { + _cohort_populations[i] = 0; + } + } + } + + static void build_cohort_populations(const double mortality_rates[], size_t cohort_populations[], const size_t cohorts) { + cohort_populations[0] = InitialPopulationSize; + for (size_t i = 1; i < cohorts; i++) { + cohort_populations[i] = cohort_populations[i - 1] * (1.0 - mortality_rates[i - 1]); + } + } +}; + +TEST_F(ShenandoahAgeCensusTest, initialize) { + const ShenandoahAgeCensus census(1); + EXPECT_EQ(census.tenuring_threshold(), ShenandoahAgeCensus::MAX_COHORTS); +} + +TEST_F(ShenandoahAgeCensusTest, ignore_small_populations) { + // Small populations are ignored so we do not return early before reaching the youngest cohort. + ShenandoahAgeCensus census(1); + add_population(census,1, 32); + add_population(census,1, 32); + census.update_census(64); + EXPECT_EQ(1u, census.tenuring_threshold()); +} + +TEST_F(ShenandoahAgeCensusTest, find_high_mortality_rate) { + ShenandoahAgeCensus census(1); + + // Initial threshold, no data + EXPECT_EQ(16u, census.tenuring_threshold()); + + // Provide population data for 1st cohort. Previous epoch has no population data so our + // algorithm skips over all cohorts, leaving tenuring threshold at 1. + update(census, 1); + EXPECT_EQ(1u, census.tenuring_threshold()); + + // Mortality rate of 1st cohort at age 1 is 0.9, we don't want to promote here. Move threshold to 2. + update(census, 2); + EXPECT_EQ(2u, census.tenuring_threshold()); + + // Mortality rate of 1st cohort at age 2 is 0.7, we don't want to promote here. Move threshold to 3. + update(census, 3); + EXPECT_EQ(3u, census.tenuring_threshold()); + + // Mortality rate of 1st cohort at age 3 is 0.5, we don't want to promote here. Move threshold to 4. + update(census, 4); + EXPECT_EQ(4u, census.tenuring_threshold()); + + // Mortality rate of 1st cohort at age 4 is 0.3, we don't want to promote here. Move threshold to 5. + update(census, 5); + EXPECT_EQ(5u, census.tenuring_threshold()); + + // Mortality rate of 1st cohort at age 5 is 0.09, this is less than the mortality rate threshold. It + // is okay to tenure objects older than 5 now. Keep threshold at 5. + update(census, 6); + EXPECT_EQ(5u, census.tenuring_threshold()); + + // Mortality rate at this age is 0. Keep tenuring threshold at 5. + update(census, 7); + EXPECT_EQ(5u, census.tenuring_threshold()); +} + +TEST_F(ShenandoahAgeCensusTest, ignore_mortality_caused_by_promotions) { + ShenandoahAgeCensus census(1); + + // Simulate a sequence of censuses with the same mortality rate. Each one will see a + // mortality rate above the tenuring threshold and raise the tenuring threshold by one. + update(census, 1); + update(census, 2); + update(census, 3); + update(census, 4); + update(census, 5); + + EXPECT_EQ(5u, census.tenuring_threshold()); + + // Simulate the effect of promoting all objects above the tenuring threshold + // out of the young generation. This will look like a very high (100%) mortality + // rate for these cohorts. However, we do _not_ want to raise the threshold in + // this case because these objects haven't really "died", they have just been + // tenured. + promote_all_tenurable(census.tenuring_threshold()); + update(census); + + // We want this to stay at 5 - the mortality in 1st cohort at age 6 was caused by expected promotions. + EXPECT_EQ(5u, census.tenuring_threshold()); +}