8380590: Parallel: Improve tenuring-threshold heuristics

Reviewed-by: gli, tschatzl

src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.cpp

@@ -41,7 +41,8 @@ PSAdaptiveSizePolicy::PSAdaptiveSizePolicy(size_t space_alignment,
      AdaptiveSizePolicy(gc_pause_goal_sec),
      _avg_promoted(new AdaptivePaddedNoZeroDevAverage(AdaptiveSizePolicyWeight, PromotedPadding)),
      _space_alignment(space_alignment),
-     _young_gen_size_increment_supplement(YoungGenerationSizeSupplement) {}
+     _young_gen_size_increment_supplement(YoungGenerationSizeSupplement),
+     _tenuring_threshold_gc_count(0) {}
 
 void PSAdaptiveSizePolicy::major_collection_begin() {
   _major_timer.reset();
@@ -223,36 +224,63 @@ size_t PSAdaptiveSizePolicy::eden_decrement_aligned_down(size_t cur_eden) {
   return align_down(eden_heap_delta, _space_alignment);
 }
 
-uint PSAdaptiveSizePolicy::compute_tenuring_threshold(bool is_survivor_overflowing,
+static const char* sizing_state_to_string(PSYoungGen::SizingState sizing_state) {
+  switch (sizing_state) {
+    case PSYoungGen::SizingState::balanced:
+      return "balanced";
+    case PSYoungGen::SizingState::constrained:
+      return "constrained";
+    case PSYoungGen::SizingState::surplus:
+      return "surplus";
+    default:
+      ShouldNotReachHere();
+      return "unknown";
+  }
+}
+
+uint PSAdaptiveSizePolicy::compute_tenuring_threshold(PSYoungGen::SizingState sizing_state,
                                                       uint tenuring_threshold) {
-  if (!young_gen_policy_is_ready()) {
+  if (AlwaysTenure || NeverTenure) {
     return tenuring_threshold;
   }
 
-  if (is_survivor_overflowing) {
-    return tenuring_threshold;
+  const uint original_threshold = tenuring_threshold;
+  constexpr uint min_tenuring_threshold = 1;
+  constexpr uint tenuring_threshold_gc_limit = 5;
+
+  switch (sizing_state) {
+    case PSYoungGen::SizingState::constrained:
+      _tenuring_threshold_gc_count = 0;
+      if (tenuring_threshold > min_tenuring_threshold) {
+        tenuring_threshold--;
+      }
+      break;
+    case PSYoungGen::SizingState::surplus:
+      if (_tenuring_threshold_gc_count < tenuring_threshold_gc_limit) {
+        _tenuring_threshold_gc_count++;
+      }
+
+      if (_tenuring_threshold_gc_count >= tenuring_threshold_gc_limit &&
+          tenuring_threshold < MaxTenuringThreshold) {
+        tenuring_threshold++;
+        _tenuring_threshold_gc_count = 0;
+      }
+      break;
+    case PSYoungGen::SizingState::balanced:
+      _tenuring_threshold_gc_count = 0;
+      break;
+    default:
+      ShouldNotReachHere();
+      break;
   }
 
-  bool incr_tenuring_threshold = false;
-
-  const double major_cost = major_gc_time_sum();
-  const double minor_cost = minor_gc_time_sum();
-
-  if (minor_cost > major_cost * _threshold_tolerance_percent) {
-    // nothing; we prefer young-gc over full-gc
-  } else if (major_cost > minor_cost * _threshold_tolerance_percent) {
-    // Major times are too long, so we want less promotion.
-    incr_tenuring_threshold = true;
-  }
-
-  // Finally, increment or decrement the tenuring threshold, as decided above.
-  // We test for decrementing first, as we might have hit the target size
-  // limit.
-  if (!(AlwaysTenure || NeverTenure)) {
-    if (incr_tenuring_threshold && tenuring_threshold < MaxTenuringThreshold) {
-      tenuring_threshold++;
-    }
-  }
+  log_debug(gc, age)("Adaptive tenuring threshold %u -> %u (max %u, young gen state: %s, increase count: %u/%u)",
+                     original_threshold,
+                     tenuring_threshold,
+                     MaxTenuringThreshold,
+                     sizing_state_to_string(sizing_state),
+                     _tenuring_threshold_gc_count,
+                     tenuring_threshold_gc_limit);
 
   return tenuring_threshold;
 }

src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.hpp

@@ -25,6 +25,7 @@
 #ifndef SHARE_GC_PARALLEL_PSADAPTIVESIZEPOLICY_HPP
 #define SHARE_GC_PARALLEL_PSADAPTIVESIZEPOLICY_HPP
 
+#include "gc/parallel/psYoungGen.hpp"
 #include "gc/shared/adaptiveSizePolicy.hpp"
 #include "gc/shared/gcUtil.hpp"
 #include "utilities/align.hpp"
@@ -46,6 +47,10 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
   // with increasing collections.
   uint _young_gen_size_increment_supplement;
 
+  // Count eligible (where eden is not squeezed by survivors) young GCs before
+  // raising the tenuring threshold.
+  uint _tenuring_threshold_gc_count;
+
   size_t decrease_eden_for_minor_pause_time(size_t current_eden_size);
 
   size_t increase_eden(size_t current_eden_size);
@@ -85,11 +90,11 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
 
   size_t compute_desired_survivor_size(size_t current_survivor_size, size_t max_gen_size);
 
-  size_t compute_old_gen_shrink_bytes(size_t old_gen_free_bytes, size_t max_shrink_bytes);
-
-  uint compute_tenuring_threshold(bool is_survivor_overflowing,
+  uint compute_tenuring_threshold(PSYoungGen::SizingState sizing_state,
                                   uint tenuring_threshold);
 
+  size_t compute_old_gen_shrink_bytes(size_t old_gen_free_bytes, size_t max_shrink_bytes);
+
   // Return the maximum size of a survivor space if the young generation were of
   // size gen_size.
   size_t max_survivor_size(size_t gen_size) {

src/hotspot/share/gc/parallel/psScavenge.cpp

@@ -333,9 +333,6 @@ bool PSScavenge::invoke(bool clear_soft_refs) {
   heap->print_before_gc();
   heap->trace_heap_before_gc(&_gc_tracer);
 
-  assert(!NeverTenure || _tenuring_threshold == markWord::max_age + 1, "Sanity");
-  assert(!AlwaysTenure || _tenuring_threshold == 0, "Sanity");
-
   // Fill in TLABs
   heap->ensure_parsability(true);  // retire TLABs
 
@@ -430,14 +427,11 @@ bool PSScavenge::invoke(bool clear_soft_refs) {
       size_policy->sample_old_gen_used_bytes(old_gen->used_in_bytes());
 
       if (UseAdaptiveSizePolicy) {
-        _tenuring_threshold = size_policy->compute_tenuring_threshold(_survivor_overflow,
-                                                                      _tenuring_threshold);
-
-        log_debug(gc, age)("New threshold %u (max threshold %u)", _tenuring_threshold, MaxTenuringThreshold);
-
         if (young_gen->is_from_to_layout()) {
           size_policy->print_stats(_survivor_overflow);
           heap->resize_after_young_gc(_survivor_overflow);
+          _tenuring_threshold = size_policy->compute_tenuring_threshold(young_gen->sizing_state(),
+                                                                        _tenuring_threshold);
         }
 
         if (UsePerfData) {
@@ -522,9 +516,9 @@ void PSScavenge::initialize() {
            "MaxTenuringThreshold should be 0 or markWord::max_age + 1, but is %d", (int) MaxTenuringThreshold);
     _tenuring_threshold = MaxTenuringThreshold;
   } else {
-    // We want to smooth out our startup times for the AdaptiveSizePolicy
-    _tenuring_threshold = (UseAdaptiveSizePolicy) ? InitialTenuringThreshold :
-                                                    MaxTenuringThreshold;
+    // We want to smooth out startup times for AdaptiveSizePolicy.
+    _tenuring_threshold = UseAdaptiveSizePolicy ? InitialTenuringThreshold
+                                                : MaxTenuringThreshold;
   }
 
   ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();

src/hotspot/share/gc/parallel/psYoungGen.cpp

@@ -43,6 +43,7 @@ PSYoungGen::PSYoungGen(ReservedSpace rs, size_t initial_size, size_t min_size, s
   _to_space(nullptr),
   _min_gen_size(min_size),
   _max_gen_size(max_size),
+  _sizing_state(SizingState::balanced),
   _gen_counters(nullptr),
   _eden_counters(nullptr),
   _from_counters(nullptr),
@@ -352,37 +353,92 @@ void PSYoungGen::compute_desired_sizes(bool is_survivor_overflowing,
   eden_size = align_up(eden_size, SpaceAlignment);
   assert(eden_size >= SpaceAlignment, "inv");
 
+  // from-space; survivor
+  const size_t survivor_used = from_space()->used_in_bytes();
+  // When survivor usage is below this ratio, consider survivor space sparse.
+  constexpr double survivor_sparse_threshold = 0.8;
+
   survivor_size = size_policy->compute_desired_survivor_size(current_survivor_size, max_gen_size());
   survivor_size = MAX3(survivor_size,
-                       from_space()->used_in_bytes(),
+                       survivor_used,
                        SpaceAlignment);
   survivor_size = align_up(survivor_size, SpaceAlignment);
 
-  log_debug(gc, ergo)("Desired size eden: %zu K, survivor: %zu K", eden_size/K, survivor_size/K);
+  log_debug(gc, ergo)("Desired size eden: %zu K, survivor: %zu K",
+                      eden_size / K,
+                      survivor_size / K);
+
+  _sizing_state = SizingState::balanced;
+
+  if (max_gen_size() < eden_size + 2 * survivor_size) {
+    log_info(gc, ergo)("Requested sizes exceed MaxNewSize (K): %zu vs %zu",
+                       (eden_size + 2 * survivor_size) / K,
+                       max_gen_size() / K);
+    // Must reduce eden/survivor to satisfy the max_gen_size constraint. Prioritize survivor_space to reduce promotion.
+    // Check if survivor is actually using its requested size.
+    if (!is_survivor_overflowing && survivor_used < survivor_sparse_threshold * survivor_size) {
+      // When survivor usage is sparse, trim survivor reservation and keep more room for eden.
+      size_t target_survivor_size = survivor_used + survivor_used / 4;
+      target_survivor_size = align_up(target_survivor_size, SpaceAlignment);
+      target_survivor_size = MAX2(target_survivor_size, SpaceAlignment);
+
+      if (target_survivor_size < survivor_size) {
+        // Decrease survivor gradually to avoid abrupt sizing swings.
+        // Simplified: new_survivor_size = survivor_size / 2 + 3 * survivor_used / 8.
+        const size_t survivor_delta = survivor_size - target_survivor_size;
+        const size_t survivor_decrement = align_up(survivor_delta / 2, SpaceAlignment);
+        survivor_size = MAX2(target_survivor_size, survivor_size - survivor_decrement);
+        log_debug(gc, ergo)("Trim survivor under MaxNewSize pressure (used: %zu K, target: %zu K, new: %zu K)",
+                            survivor_used / K,
+                            target_survivor_size / K,
+                            survivor_size / K);
+      }
+    }
+
+    // Recheck after potential survivor_size adjustment.
+    if (max_gen_size() < eden_size + 2 * survivor_size) {
+      if (2 * survivor_size >= max_gen_size()) {
+        // If requested survivor size is too large
+        survivor_size = align_down((max_gen_size() - SpaceAlignment) / 2, SpaceAlignment);
+      }
 
-  const size_t new_gen_size = eden_size + 2 * survivor_size;
-  if (new_gen_size < min_gen_size()) {
-    // Keep survivor and adjust eden to meet min-gen-size
-    eden_size = min_gen_size() - 2 * survivor_size;
-  } else if (max_gen_size() < new_gen_size) {
-    log_info(gc, ergo)("Requested sizes exceeds MaxNewSize (K): %zu vs %zu", new_gen_size/K, max_gen_size()/K);
-    // New capacity would exceed max; need to revise these desired sizes.
-    // Favor survivor over eden in order to reduce promotion (overflow).
-    if (2 * survivor_size >= max_gen_size()) {
-      // If requested survivor size is too large
-      survivor_size = align_down((max_gen_size() - SpaceAlignment) / 2, SpaceAlignment);
-      eden_size = max_gen_size() - 2 * survivor_size;
-    } else {
       // Respect survivor size and reduce eden
       eden_size = max_gen_size() - 2 * survivor_size;
+
+      _sizing_state = SizingState::constrained;
     }
   }
 
-  assert(eden_size >= SpaceAlignment, "inv");
-  assert(survivor_size >= SpaceAlignment, "inv");
+  if (eden_size + 2 * survivor_size < min_gen_size()) {
+    // Keep survivor and adjust eden to meet min-gen-size.
+    eden_size = min_gen_size() - 2 * survivor_size;
 
-  assert(is_aligned(eden_size, SpaceAlignment), "inv");
-  assert(is_aligned(survivor_size, SpaceAlignment), "inv");
+    _sizing_state = SizingState::surplus;
+  }
+
+  const size_t final_gen_size = eden_size + 2 * survivor_size;
+  // A balanced result fills max_gen_size; otherwise there is surplus young-gen headroom.
+  if (_sizing_state == SizingState::balanced) {
+    if (final_gen_size < max_gen_size()) {
+      _sizing_state = SizingState::surplus;
+    }
+  }
+
+#ifdef ASSERT
+  {
+    assert(eden_size >= SpaceAlignment, "inv");
+    assert(survivor_size >= SpaceAlignment, "inv");
+
+    assert(is_aligned(eden_size, SpaceAlignment), "inv");
+    assert(is_aligned(survivor_size, SpaceAlignment), "inv");
+
+    assert(final_gen_size >= min_gen_size(), "inv");
+    assert(final_gen_size <= max_gen_size(), "inv");
+    if (final_gen_size < max_gen_size()) {
+      assert(_sizing_state == SizingState::surplus, "inv");
+    }
+  }
+#endif
 }
 
 void PSYoungGen::resize_inner(size_t desired_eden_size,

src/hotspot/share/gc/parallel/psYoungGen.hpp

@@ -37,6 +37,22 @@ class PSYoungGen : public CHeapObj<mtGC> {
   friend class VMStructs;
   friend class ParallelScavengeHeap;
 
+ public:
+  // Young generation sizing state from the latest sizing pass. It records how
+  // the desired eden/survivor sizes relate to the young-gen size bounds.
+  // Consumers such as the tenuring-threshold heuristic can use this as sizing
+  // feedback.
+  enum class SizingState : int {
+    // Desired young-gen size means "eden + 2 * survivor".
+    // Its relation to max_gen_size is:
+    // exactly equal.
+    balanced = 0,
+    // greater than.
+    constrained,
+    // less than.
+    surplus
+  };
+
  private:
   MemRegion       _reserved;
   PSVirtualSpace* _virtual_space;
@@ -50,6 +66,9 @@ class PSYoungGen : public CHeapObj<mtGC> {
   const size_t _min_gen_size;
   const size_t _max_gen_size;
 
+  // Current young-gen sizing state, updated by compute_desired_sizes().
+  SizingState _sizing_state;
+
   // Performance counters
   GenerationCounters*   _gen_counters;
   HSpaceCounters*       _eden_counters;
@@ -127,6 +146,8 @@ class PSYoungGen : public CHeapObj<mtGC> {
   size_t min_gen_size() const { return _min_gen_size; }
   size_t max_gen_size() const { return _max_gen_size; }
 
+  SizingState sizing_state() const { return _sizing_state; }
+
   // Allocation
   HeapWord* cas_allocate(size_t word_size) {
     HeapWord* result = eden_space()->cas_allocate(word_size);

src/hotspot/share/gc/shared/adaptiveSizePolicy.cpp

@@ -43,8 +43,7 @@ AdaptiveSizePolicy::AdaptiveSizePolicy(double gc_pause_goal_sec) :
   _peak_old_used_bytes_seq(seq_default_alpha_value),
   _minor_pause_young_estimator(new LinearLeastSquareFit(AdaptiveSizePolicyWeight)),
   _threshold_tolerance_percent(1.0 + ThresholdTolerance/100.0),
-  _gc_pause_goal_sec(gc_pause_goal_sec),
-  _young_gen_policy_is_ready(false) {}
+  _gc_pause_goal_sec(gc_pause_goal_sec) {}
 
 void AdaptiveSizePolicy::minor_collection_begin() {
   _minor_timer.reset();
@@ -61,12 +60,6 @@ void AdaptiveSizePolicy::minor_collection_end(size_t eden_capacity_in_bytes) {
   record_gc_duration(minor_pause_in_seconds);
   _trimmed_minor_gc_time_seconds.add(minor_pause_in_seconds);
 
-  if (!_young_gen_policy_is_ready) {
-    // The policy does not have enough data until at least some
-    // young collections have been done.
-    _young_gen_policy_is_ready = GCId::current() >= AdaptiveSizePolicyReadyThreshold;
-  }
-
   {
     double eden_size_in_mbytes = ((double)eden_capacity_in_bytes)/((double)M);
     _minor_pause_young_estimator->update(eden_size_in_mbytes, minor_pause_in_ms);

src/hotspot/share/gc/shared/adaptiveSizePolicy.hpp

@@ -133,9 +133,6 @@ class AdaptiveSizePolicy : public CHeapObj<mtGC> {
 
   const double _gc_pause_goal_sec; // Goal for maximum GC pause
 
-  // Flag indicating that the adaptive policy is ready to use
-  bool _young_gen_policy_is_ready;
-
   // Accessors
   double gc_pause_goal_sec() const { return _gc_pause_goal_sec; }
 
@@ -160,8 +157,6 @@ class AdaptiveSizePolicy : public CHeapObj<mtGC> {
     return gc_percent;
   }
 
-  bool young_gen_policy_is_ready() { return _young_gen_policy_is_ready; }
-
   size_t eden_increment(size_t cur_eden);
   size_t eden_increment(size_t cur_eden, uint percent_change);