8384221: GenShen: WhiteBox full GC promotion is not reliable

Reviewed-by: wkemper, ysr

src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp

@@ -358,7 +358,7 @@ size_t ShenandoahGenerationalHeuristics::select_aged_regions(ShenandoahInPlacePr
 
 // Having chosen the collection set, adjust the budgets for generational mode based on its composition.  Note
 // that young_generation->available() now knows about recently discovered immediate garbage.
-void ShenandoahGenerationalHeuristics::adjust_evacuation_budgets(ShenandoahHeap* const heap,
+void ShenandoahGenerationalHeuristics::adjust_evacuation_budgets(ShenandoahGenerationalHeap* const heap,
                                                                  ShenandoahCollectionSet* const collection_set) {
   shenandoah_assert_generational();
   // We may find that old_evacuation_reserve and/or loaned_for_young_evacuation are not fully consumed, in which case we may
@@ -481,6 +481,16 @@ void ShenandoahGenerationalHeuristics::adjust_evacuation_budgets(ShenandoahHeap*
 
   if (add_regions_to_young > 0) {
     assert(excess_old >= add_regions_to_young * region_size_bytes, "Cannot xfer more than excess old");
+    if (heap->age_census()->is_always_tenure()) {
+      // Cap excess_old at one min-PLAB per worker so this much stays in old's promotion reserve
+      // instead of being transferred to young.
+      const size_t min_plab_total = heap->plab_min_size() * HeapWordSize * heap->workers()->max_workers();
+      if (excess_old > min_plab_total) {
+        excess_old = min_plab_total;
+        // Avoid underflowing excess_old when we subtract below.
+        add_regions_to_young = 0;
+      }
+    }
     excess_old -= add_regions_to_young * region_size_bytes;
     log_debug(gc, ergo)("Before start of evacuation, total_promotion reserve is young_advance_promoted_reserve: %zu "
                         "plus excess: old: %zu", young_advance_promoted_reserve_used, excess_old);

src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.hpp

@@ -92,7 +92,7 @@ private:
 
   // Adjust evacuation budgets after choosing collection set.  On entry, the instance variable _regions_to_xfer
   // represents regions to be transferred to old based on decisions made in top_off_collection_set()
-  void adjust_evacuation_budgets(ShenandoahHeap* const heap,
+  void adjust_evacuation_budgets(ShenandoahGenerationalHeap* const heap,
                                  ShenandoahCollectionSet* const collection_set);
 
 protected:

src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp

@@ -179,6 +179,12 @@ void ShenandoahGlobalHeuristics::choose_global_collection_set(ShenandoahCollecti
   size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + original_young_evac_reserve;
   size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0;
 
+  // Admit every region with any garbage so every live object gets a chance to be promoted.
+  if (heap->age_census()->is_always_tenure()) {
+    ignore_threshold = 0;
+    min_garbage = SIZE_MAX;
+  }
+
   ShenandoahGlobalCSetBudget budget(region_size_bytes,
                                     shared_reserve_regions * region_size_bytes,
                                     garbage_threshold, ignore_threshold, min_garbage,

src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp

@@ -34,7 +34,7 @@ ShenandoahAgeCensus::ShenandoahAgeCensus()
 }
 
 ShenandoahAgeCensus::ShenandoahAgeCensus(uint max_workers)
-  : _max_workers(max_workers)
+  : _max_workers(max_workers), _always_tenure(false)
 {
   if (ShenandoahGenerationalMinTenuringAge > ShenandoahGenerationalMaxTenuringAge) {
     vm_exit_during_initialization(

src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.hpp

@@ -121,6 +121,8 @@ class ShenandoahAgeCensus: public CHeapObj<mtGC> {
 
   uint _max_workers;                 // Maximum number of workers for parallel tasks
 
+  bool _always_tenure;               // When true, every age is tenurable.
+
   // Mortality rate of a cohort, given its population in
   // previous and current epochs
   double mortality_rate(size_t prev_pop, size_t cur_pop);
@@ -150,9 +152,9 @@ class ShenandoahAgeCensus: public CHeapObj<mtGC> {
     return _tenuring_threshold[prev];
   }
 
-  // Override the tenuring threshold for the current epoch. This is used to
-  // cause everything to be promoted for a whitebox full gc request.
-  void set_tenuring_threshold(uint threshold) { _tenuring_threshold[_epoch] = threshold; }
+  // Set always tenure mode. Currently only used by ShenandoahTenuringOverride
+  // to force is_tenurable() to be true for every age during WB.fullGC tests.
+  void set_always_tenure(bool always_tenure) { _always_tenure = always_tenure; }
 
 #ifndef PRODUCT
   // Return the sum of size of objects of all ages recorded in the
@@ -187,11 +189,13 @@ class ShenandoahAgeCensus: public CHeapObj<mtGC> {
   // Visible for testing. Use is_tenurable for consistent tenuring comparisons.
   uint tenuring_threshold() const { return _tenuring_threshold[_epoch]; }
 
-  // Return true if this age is at or above the tenuring threshold.
+  // Return true if this age is at or above the tenuring threshold, or if always tenure is enabled.
   bool is_tenurable(uint age) const {
-    return age >= tenuring_threshold();
+    return age >= tenuring_threshold() || _always_tenure;
   }
 
+  bool is_always_tenure() const { return _always_tenure; }
+
   // Update the local age table for worker_id by size for
   // given obj_age, region_age, and region_youth
   CENSUS_NOISE(void add(uint obj_age, uint region_age, uint region_youth, size_t size, uint worker_id);)
@@ -244,24 +248,22 @@ class ShenandoahAgeCensus: public CHeapObj<mtGC> {
   void print();
 };
 
-// RAII object that temporarily overrides the tenuring threshold for the
-// duration of a scope, restoring the original value on destruction.
-// Used to force promotion of all young objects during whitebox full GCs.
+// RAII object that enables ShenandoahAgeCensus always tenure mode for the
+// duration of a scope and disables it on destruction. Used to force promotion
+// of all young objects during whitebox full GCs.
 class ShenandoahTenuringOverride : public StackObj {
   ShenandoahAgeCensus* _census;
-  uint _saved_threshold;
   bool _active;
 public:
   ShenandoahTenuringOverride(bool active, ShenandoahAgeCensus* census) :
-    _census(census), _saved_threshold(0), _active(active) {
+    _census(census), _active(active) {
     if (_active) {
-      _saved_threshold = _census->tenuring_threshold();
-      _census->set_tenuring_threshold(0);
+      _census->set_always_tenure(true);
     }
   }
   ~ShenandoahTenuringOverride() {
     if (_active) {
-      _census->set_tenuring_threshold(_saved_threshold);
+      _census->set_always_tenure(false);
     }
   }
 };

src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp

@@ -269,7 +269,7 @@ void ShenandoahGenerationalControlThread::run_gc_cycle(const ShenandoahGCRequest
     // Cannot uncommit bitmap slices during concurrent reset
     ShenandoahNoUncommitMark forbid_region_uncommit(_heap);
 
-    // When a whitebox full GC is requested, set the tenuring threshold to zero
+    // When a whitebox full GC is requested, set the age census to always tenure
     // so that all young objects are promoted to old. This ensures that tests
     // using WB.fullGC() to promote objects to old gen will not loop forever.
     ShenandoahTenuringOverride tenuring_override(request.cause == GCCause::_wb_full_gc,

src/hotspot/share/gc/shenandoah/shenandoahPLAB.cpp

@@ -39,7 +39,7 @@ ShenandoahPLAB::ShenandoahPLAB() :
   _promoted(0),
   _promotion_failure_count(0),
   _promotion_failure_words(0),
-  _allows_promotion(false),
+  _allows_promotion(true),
   _retries_enabled(false),
   _heap(ShenandoahGenerationalHeap::heap()) {
   _plab = new PLAB(PLAB::min_size());