8262185: G1: Prune collection set candidates early

Reviewed-by: iwalulya, sjohanss, ayang

src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -35,6 +35,25 @@ void G1CollectionSetCandidates::remove(uint num_regions) {
   }
 }
 
+void G1CollectionSetCandidates::remove_from_end(uint num_remove, size_t wasted) {
+  assert(num_remove <= num_remaining(), "trying to remove more regions than remaining");
+
+#ifdef ASSERT
+  size_t reclaimable = 0;
+
+  for (uint i = 0; i < num_remove; i++) {
+    uint cur_idx = _num_regions - i - 1;
+    reclaimable += at(cur_idx)->reclaimable_bytes();
+    // Make sure we crash if we access it.
+    _regions[cur_idx] = NULL;
+  }
+
+  assert(reclaimable == wasted, "Recalculated reclaimable inconsistent");
+#endif
+  _num_regions -= num_remove;
+  _remaining_reclaimable_bytes -= wasted;
+}
+
 void G1CollectionSetCandidates::iterate(HeapRegionClosure* cl) {
   for (uint i = _front_idx; i < _num_regions; i++) {
     HeapRegion* r = _regions[i];
@@ -45,6 +64,16 @@ void G1CollectionSetCandidates::iterate(HeapRegionClosure* cl) {
   }
 }
 
+void G1CollectionSetCandidates::iterate_backwards(HeapRegionClosure* cl) {
+  for (uint i = _num_regions; i > _front_idx; i--) {
+    HeapRegion* r = _regions[i - 1];
+    if (cl->do_heap_region(r)) {
+      cl->set_incomplete();
+      break;
+    }
+  }
+}
+
 #ifndef PRODUCT
 void G1CollectionSetCandidates::verify() const {
   guarantee(_front_idx <= _num_regions, "Index: %u Num_regions: %u", _front_idx, _num_regions);

src/hotspot/share/gc/g1/g1CollectionSetCandidates.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -74,10 +74,16 @@ public:
     return res;
   }
 
+  // Remove num_regions from the front of the collection set candidate list.
   void remove(uint num_regions);
+  // Remove num_remove regions from the back of the collection set candidate list.
+  void remove_from_end(uint num_remove, size_t wasted);
 
   // Iterate over all remaining collection set candidate regions.
   void iterate(HeapRegionClosure* cl);
+  // Iterate over all remaining collectin set candidate regions from the end
+  // to the beginning of the set.
+  void iterate_backwards(HeapRegionClosure* cl);
 
   // Return the number of candidate regions remaining.
   uint num_remaining() { return _num_regions - _front_idx; }

src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -257,6 +257,60 @@ bool G1CollectionSetChooser::should_add(HeapRegion* hr) {
          hr->rem_set()->is_complete();
 }
 
+// Closure implementing early pruning (removal) of regions meeting the
+// G1HeapWastePercent criteria. That is, either until _max_pruned regions were
+// removed (for forward progress in evacuation) or the waste accumulated by the
+// removed regions is above max_wasted.
+class G1PruneRegionClosure : public HeapRegionClosure {
+  uint _num_pruned;
+  size_t _cur_wasted;
+
+  uint const _max_pruned;
+  size_t const _max_wasted;
+
+public:
+  G1PruneRegionClosure(uint max_pruned, size_t max_wasted) :
+    _num_pruned(0), _cur_wasted(0), _max_pruned(max_pruned), _max_wasted(max_wasted) { }
+
+  virtual bool do_heap_region(HeapRegion* r) {
+    size_t const reclaimable = r->reclaimable_bytes();
+    if (_num_pruned > _max_pruned ||
+        _cur_wasted + reclaimable > _max_wasted) {
+      return true;
+    }
+    r->rem_set()->clear(true /* cardset_only */);
+    _cur_wasted += reclaimable;
+    _num_pruned++;
+    return false;
+  }
+
+  uint num_pruned() const { return _num_pruned; }
+  size_t wasted() const { return _cur_wasted; }
+};
+
+void G1CollectionSetChooser::prune(G1CollectionSetCandidates* candidates) {
+  G1Policy* p = G1CollectedHeap::heap()->policy();
+
+  uint min_old_cset_length = p->calc_min_old_cset_length(candidates);
+  uint num_candidates = candidates->num_regions();
+
+  if (min_old_cset_length < num_candidates) {
+    size_t allowed_waste = p->allowed_waste_in_collection_set();
+
+    G1PruneRegionClosure prune_cl(num_candidates - min_old_cset_length,
+                                  allowed_waste);
+    candidates->iterate_backwards(&prune_cl);
+
+    log_debug(gc, ergo, cset)("Pruned %u regions out of %u, leaving " SIZE_FORMAT " bytes waste (allowed " SIZE_FORMAT ")",
+                              prune_cl.num_pruned(),
+                              candidates->num_regions(),
+                              prune_cl.wasted(),
+                              allowed_waste);
+
+    candidates->remove_from_end(prune_cl.num_pruned(), prune_cl.wasted());
+  }
+}
+
 G1CollectionSetCandidates* G1CollectionSetChooser::build(WorkGang* workers, uint max_num_regions) {
   uint num_workers = workers->active_workers();
   uint chunk_size = calculate_work_chunk_size(num_workers, max_num_regions);
@@ -265,6 +319,7 @@ G1CollectionSetCandidates* G1CollectionSetChooser::build(WorkGang* workers, uint
   workers->run_task(&cl, num_workers);
 
   G1CollectionSetCandidates* result = cl.get_sorted_candidates();
+  prune(result);
   result->verify();
   return result;
 }

src/hotspot/share/gc/g1/g1CollectionSetChooser.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -36,6 +36,11 @@ class WorkGang;
 // methods.
 class G1CollectionSetChooser : public AllStatic {
   static uint calculate_work_chunk_size(uint num_workers, uint num_regions);
+
+  // Remove regions in the collection set candidates as long as the G1HeapWastePercent
+  // criteria is met. Keep at least the minimum amount of old regions to guarantee
+  // some progress.
+  static void prune(G1CollectionSetCandidates* candidates);
 public:
 
   static size_t mixed_gc_live_threshold_bytes() {

src/hotspot/share/gc/g1/g1Policy.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1278,22 +1278,16 @@ bool G1Policy::next_gc_should_be_mixed(const char* true_action_str,
     log_debug(gc, ergo)("%s (candidate old regions not available)", false_action_str);
     return false;
   }
-
-  // Is the amount of uncollected reclaimable space above G1HeapWastePercent?
-  size_t reclaimable_bytes = candidates->remaining_reclaimable_bytes();
-  double reclaimable_percent = reclaimable_bytes_percent(reclaimable_bytes);
-  double threshold = (double) G1HeapWastePercent;
-  if (reclaimable_percent <= threshold) {
-    log_debug(gc, ergo)("%s (reclaimable percentage not over threshold). candidate old regions: %u reclaimable: " SIZE_FORMAT " (%1.2f) threshold: " UINTX_FORMAT,
-                        false_action_str, candidates->num_remaining(), reclaimable_bytes, reclaimable_percent, G1HeapWastePercent);
-    return false;
-  }
-  log_debug(gc, ergo)("%s (candidate old regions available). candidate old regions: %u reclaimable: " SIZE_FORMAT " (%1.2f) threshold: " UINTX_FORMAT,
-                      true_action_str, candidates->num_remaining(), reclaimable_bytes, reclaimable_percent, G1HeapWastePercent);
+  // Go through all regions - we already pruned regions not worth collecting
+  // during candidate selection.
   return true;
 }
 
-uint G1Policy::calc_min_old_cset_length() const {
+size_t G1Policy::allowed_waste_in_collection_set() const {
+  return G1HeapWastePercent * _g1h->capacity() / 100;
+}
+
+uint G1Policy::calc_min_old_cset_length(G1CollectionSetCandidates* candidates) const {
   // The min old CSet region bound is based on the maximum desired
   // number of mixed GCs after a cycle. I.e., even if some old regions
   // look expensive, we should add them to the CSet anyway to make
@@ -1304,7 +1298,7 @@ uint G1Policy::calc_min_old_cset_length() const {
   // to the CSet candidates in the first place, not how many remain, so
   // that the result is the same during all mixed GCs that follow a cycle.
 
-  const size_t region_num = _collection_set->candidates()->num_regions();
+  const size_t region_num = candidates->num_regions();
   const size_t gc_num = (size_t) MAX2(G1MixedGCCountTarget, (uintx) 1);
   size_t result = region_num / gc_num;
   // emulate ceiling
@@ -1347,7 +1341,7 @@ void G1Policy::calculate_old_collection_set_regions(G1CollectionSetCandidates* c
 
   double optional_threshold_ms = time_remaining_ms * optional_prediction_fraction();
 
-  const uint min_old_cset_length = calc_min_old_cset_length();
+  const uint min_old_cset_length = calc_min_old_cset_length(candidates);
   const uint max_old_cset_length = MAX2(min_old_cset_length, calc_max_old_cset_length());
   const uint max_optional_regions = max_old_cset_length - min_old_cset_length;
   bool check_time_remaining = use_adaptive_young_list_length();

src/hotspot/share/gc/g1/g1Policy.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -246,7 +246,7 @@ public:
 
   // Calculate the minimum number of old regions we'll add to the CSet
   // during a mixed GC.
-  uint calc_min_old_cset_length() const;
+  uint calc_min_old_cset_length(G1CollectionSetCandidates* candidates) const;
 
   // Calculate the maximum number of old regions we'll add to the CSet
   // during a mixed GC.
@@ -347,6 +347,8 @@ public:
   bool next_gc_should_be_mixed(const char* true_action_str,
                                const char* false_action_str) const;
 
+  // Amount of allowed waste in bytes in the collection set.
+  size_t allowed_waste_in_collection_set() const;
   // Calculate and return the number of initial and optional old gen regions from
   // the given collection set candidates and the remaining time.
   void calculate_old_collection_set_regions(G1CollectionSetCandidates* candidates,