8277736: G1: Allow forced evacuation failure of first N regions in collection set

Reviewed-by: ayang, tschatzl

src/hotspot/share/gc/g1/g1ParScanThreadState.cpp

@@ -421,8 +421,8 @@ HeapWord* G1ParScanThreadState::allocate_copy_slow(G1HeapRegionAttr* dest_attr,
 }
 
 #if EVAC_FAILURE_INJECTOR
-bool G1ParScanThreadState::inject_evacuation_failure() {
-  return _g1h->evac_failure_injector()->evacuation_should_fail(_evac_failure_inject_counter);
+bool G1ParScanThreadState::inject_evacuation_failure(uint region_idx) {
+  return _g1h->evac_failure_injector()->evacuation_should_fail(_evac_failure_inject_counter, region_idx);
 }
 #endif
 
@@ -470,7 +470,7 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
   assert(_g1h->is_in_reserved(obj_ptr), "Allocated memory should be in the heap");
 
   // Should this evacuation fail?
-  if (inject_evacuation_failure()) {
+  if (inject_evacuation_failure(from_region->hrm_index())) {
     // Doing this after all the allocation attempts also tests the
     // undo_allocation() method too.
     undo_allocation(dest_attr, obj_ptr, word_sz, node_index);

src/hotspot/share/gc/g1/g1ParScanThreadState.hpp

@@ -107,7 +107,7 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
   EvacuationFailedInfo _evacuation_failed_info;
   G1EvacFailureRegions* _evac_failure_regions;
 
-  bool inject_evacuation_failure() EVAC_FAILURE_INJECTOR_RETURN_( return false; );
+  bool inject_evacuation_failure(uint region_idx) EVAC_FAILURE_INJECTOR_RETURN_( return false; );
 
 public:
   G1ParScanThreadState(G1CollectedHeap* g1h,

src/hotspot/share/gc/g1/g1YoungGCEvacFailureInjector.cpp

@@ -30,6 +30,33 @@
 
 #if EVAC_FAILURE_INJECTOR
 
+class SelectEvacFailureRegionClosure : public HeapRegionClosure {
+  CHeapBitMap& _evac_failure_regions;
+  size_t _evac_failure_regions_num;
+
+public:
+  SelectEvacFailureRegionClosure(CHeapBitMap& evac_failure_regions, size_t cset_length) :
+    _evac_failure_regions(evac_failure_regions),
+    _evac_failure_regions_num(cset_length * G1EvacuationFailureALotCSetPercent / 100) { }
+
+  bool do_heap_region(HeapRegion* r) override {
+    assert(r->in_collection_set(), "must be");
+    if (_evac_failure_regions_num > 0) {
+      _evac_failure_regions.set_bit(r->hrm_index());
+      --_evac_failure_regions_num;
+      return false;
+    }
+    return true;
+  }
+};
+
+void G1YoungGCEvacFailureInjector::select_evac_failure_regions() {
+  G1CollectedHeap* g1h = G1CollectedHeap::heap();
+  _evac_failure_regions.reinitialize(g1h->max_reserved_regions());
+  SelectEvacFailureRegionClosure closure(_evac_failure_regions, g1h->collection_set()->cur_length());
+  g1h->collection_set_iterate_all(&closure);
+}
+
 bool G1YoungGCEvacFailureInjector::arm_if_needed_for_gc_type(bool for_young_gc,
                                                              bool during_concurrent_start,
                                                              bool mark_or_rebuild_in_progress) {
@@ -68,6 +95,10 @@ void G1YoungGCEvacFailureInjector::arm_if_needed() {
       arm_if_needed_for_gc_type(in_young_only_phase,
                                 in_concurrent_start_gc,
                                 mark_or_rebuild_in_progress);
+
+    if (_inject_evacuation_failure_for_current_gc) {
+      select_evac_failure_regions();
+    }
   }
 }
 

src/hotspot/share/gc/g1/g1YoungGCEvacFailureInjector.hpp

@@ -56,11 +56,17 @@ class G1YoungGCEvacFailureInjector {
   // Used to determine whether evacuation failure injection should be in effect
   // for the current GC.
   size_t _last_collection_with_evacuation_failure;
+
+  // Records the regions that will fail evacuation.
+  CHeapBitMap _evac_failure_regions;
 #endif
 
   bool arm_if_needed_for_gc_type(bool for_young_gc,
                                  bool during_concurrent_start,
                                  bool mark_or_rebuild_in_progress) EVAC_FAILURE_INJECTOR_RETURN_( return false; );
+
+  // Selects the regions that will fail evacuation by G1EvacuationFailureALotCSetPercent.
+  void select_evac_failure_regions() EVAC_FAILURE_INJECTOR_RETURN;
 public:
 
   // Arm the evacuation failure injector if needed for the current
@@ -69,7 +75,7 @@ public:
 
   // Return true if it's time to cause an evacuation failure; the caller
   // provides the (preferably thread-local) counter to minimize performance impact.
-  bool evacuation_should_fail(size_t& counter) EVAC_FAILURE_INJECTOR_RETURN_( return false; );
+  bool evacuation_should_fail(size_t& counter, uint region_idx) EVAC_FAILURE_INJECTOR_RETURN_( return false; );
 
   // Reset the evacuation failure injection counters. Should be called at
   // the end of an evacuation pause in which an evacuation failure occurred.

src/hotspot/share/gc/g1/g1YoungGCEvacFailureInjector.inline.hpp

@@ -32,10 +32,13 @@
 
 #if EVAC_FAILURE_INJECTOR
 
-inline bool G1YoungGCEvacFailureInjector::evacuation_should_fail(size_t& counter) {
+inline bool G1YoungGCEvacFailureInjector::evacuation_should_fail(size_t& counter, uint region_idx) {
   if (!_inject_evacuation_failure_for_current_gc) {
     return false;
   }
+  if (!_evac_failure_regions.at(region_idx)) {
+    return false;
+  }
   if (++counter < G1EvacuationFailureALotCount) {
     return false;
   }

src/hotspot/share/gc/g1/g1_globals.hpp

@@ -73,7 +73,13 @@
                                                                             \
   product(bool, G1EvacuationFailureALotDuringMixedGC, true,                 \
           "Force use of evacuation failure handling during mixed "          \
-          "evacuation pauses")
+          "evacuation pauses")                                              \
+                                                                            \
+  product(uint, G1EvacuationFailureALotCSetPercent, 100,                    \
+          "The percentage of regions in the collection set starting "       \
+          "from the beginning where the forced evacuation failure "         \
+          "injection will be applied.")                                     \
+          range(1, 100)
 #else
 #define GC_G1_EVACUATION_FAILURE_FLAGS(develop,                             \
                                        develop_pd,                          \