8272093: Extract evacuation failure injection from G1CollectedHeap

Reviewed-by: ayang, sjohanss

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -68,6 +68,7 @@
 #include "gc/g1/g1ServiceThread.hpp"
 #include "gc/g1/g1UncommitRegionTask.hpp"
 #include "gc/g1/g1VMOperations.hpp"
+#include "gc/g1/g1YoungGCEvacFailureInjector.hpp"
 #include "gc/g1/g1YoungGCPostEvacuateTasks.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
 #include "gc/g1/heapRegionRemSet.inline.hpp"
@@ -1448,6 +1449,7 @@ G1CollectedHeap::G1CollectedHeap() :
   _numa(G1NUMA::create()),
   _hrm(),
   _allocator(NULL),
+  _evac_failure_injector(),
   _verifier(NULL),
   _summary_bytes_used(0),
   _bytes_used_during_gc(0),
@@ -1478,11 +1480,6 @@ G1CollectedHeap::G1CollectedHeap() :
   _task_queues(NULL),
   _num_regions_failed_evacuation(0),
   _regions_failed_evacuation(mtGC),
-#ifndef PRODUCT
-  _evacuation_failure_alot_for_current_gc(false),
-  _evacuation_failure_alot_gc_number(0),
-  _evacuation_failure_alot_count(0),
-#endif
   _ref_processor_stw(NULL),
   _is_alive_closure_stw(this),
   _is_subject_to_discovery_stw(this),
@@ -1511,8 +1508,6 @@ G1CollectedHeap::G1CollectedHeap() :
     _task_queues->register_queue(i, q);
   }
 
-  // Initialize the G1EvacuationFailureALot counters and flags.
-  NOT_PRODUCT(reset_evacuation_should_fail();)
   _gc_tracer_stw->initialize();
 
   guarantee(_task_queues != NULL, "task_queues allocation failure.");
@@ -1767,6 +1762,8 @@ jint G1CollectedHeap::initialize() {
 
   _regions_failed_evacuation.resize(max_regions());
 
+  evac_failure_injector()->reset();
+
   G1InitLogger::print();
 
   return JNI_OK;
@@ -3583,7 +3580,7 @@ void G1CollectedHeap::pre_evacuate_collection_set(G1EvacuationInfo* evacuation_i
   }
 
   // Should G1EvacuationFailureALot be in effect for this GC?
-  NOT_PRODUCT(set_evacuation_failure_alot_for_current_gc();)
+  evac_failure_injector()->arm_if_needed();
 }
 
 class G1EvacuateRegionsBaseTask : public AbstractGangTask {
@@ -4298,7 +4295,7 @@ void G1CollectedHeap::unregister_nmethod(nmethod* nm) {
 void G1CollectedHeap::update_used_after_gc() {
   if (evacuation_failed()) {
     // Reset the G1EvacuationFailureALot counters and flags
-    NOT_PRODUCT(reset_evacuation_should_fail();)
+    evac_failure_injector()->reset();
 
     set_used(recalculate_used());
 

src/hotspot/share/gc/g1/g1CollectedHeap.hpp

@@ -47,6 +47,7 @@
 #include "gc/g1/g1NUMA.hpp"
 #include "gc/g1/g1RedirtyCardsQueue.hpp"
 #include "gc/g1/g1SurvivorRegions.hpp"
+#include "gc/g1/g1YoungGCEvacFailureInjector.hpp"
 #include "gc/g1/heapRegionManager.hpp"
 #include "gc/g1/heapRegionSet.hpp"
 #include "gc/shared/barrierSet.hpp"
@@ -211,6 +212,8 @@ private:
   // Manages all allocations with regions except humongous object allocations.
   G1Allocator* _allocator;
 
+  G1YoungGCEvacFailureInjector _evac_failure_injector;
+
   // Manages all heap verification.
   G1HeapVerifier* _verifier;
 
@@ -562,6 +565,8 @@ public:
     return _allocator;
   }
 
+  G1YoungGCEvacFailureInjector* evac_failure_injector() { return &_evac_failure_injector; }
+
   G1HeapVerifier* verifier() {
     return _verifier;
   }
@@ -871,38 +876,9 @@ public:
   // Records for every region on the heap whether evacuation failed for it.
   CHeapBitMap _regions_failed_evacuation;
 
-#ifndef PRODUCT
-  // Support for forcing evacuation failures. Analogous to
-  // PromotionFailureALot for the other collectors.
-
-  // Records whether G1EvacuationFailureALot should be in effect
-  // for the current GC
-  bool _evacuation_failure_alot_for_current_gc;
-
-  // Used to record the GC number for interval checking when
-  // determining whether G1EvaucationFailureALot is in effect
-  // for the current GC.
-  size_t _evacuation_failure_alot_gc_number;
-
-  // Count of the number of evacuations between failures.
-  volatile size_t _evacuation_failure_alot_count;
-
-  // Set whether G1EvacuationFailureALot should be in effect
-  // for the current GC (based upon the type of GC and which
-  // command line flags are set);
-  inline bool evacuation_failure_alot_for_gc_type(bool for_young_gc,
-                                                  bool during_concurrent_start,
-                                                  bool mark_or_rebuild_in_progress);
-
-  inline void set_evacuation_failure_alot_for_current_gc();
-
-  // Return true if it's time to cause an evacuation failure.
-  inline bool evacuation_should_fail();
-
-  // Reset the G1EvacuationFailureALot counters.  Should be called at
-  // the end of an evacuation pause in which an evacuation failure occurred.
-  inline void reset_evacuation_should_fail();
-#endif // !PRODUCT
+  // Preserve the mark of "obj", if necessary, in preparation for its mark
+  // word being overwritten with a self-forwarding-pointer.
+  void preserve_mark_during_evac_failure(uint worker_id, oop obj, markWord m);
 
   // ("Weak") Reference processing support.
   //

src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp

@@ -211,78 +211,6 @@ bool G1CollectedHeap::notify_region_failed_evacuation(uint const region_idx) {
   return result;
 }
 
-#ifndef PRODUCT
-// Support for G1EvacuationFailureALot
-
-inline bool
-G1CollectedHeap::evacuation_failure_alot_for_gc_type(bool for_young_gc,
-                                                     bool during_concurrent_start,
-                                                     bool mark_or_rebuild_in_progress) {
-  bool res = false;
-  if (mark_or_rebuild_in_progress) {
-    res |= G1EvacuationFailureALotDuringConcMark;
-  }
-  if (during_concurrent_start) {
-    res |= G1EvacuationFailureALotDuringConcurrentStart;
-  }
-  if (for_young_gc) {
-    res |= G1EvacuationFailureALotDuringYoungGC;
-  } else {
-    // GCs are mixed
-    res |= G1EvacuationFailureALotDuringMixedGC;
-  }
-  return res;
-}
-
-inline void
-G1CollectedHeap::set_evacuation_failure_alot_for_current_gc() {
-  if (G1EvacuationFailureALot) {
-    // Note we can't assert that _evacuation_failure_alot_for_current_gc
-    // is clear here. It may have been set during a previous GC but that GC
-    // did not copy enough objects (i.e. G1EvacuationFailureALotCount) to
-    // trigger an evacuation failure and clear the flags and and counts.
-
-    // Check if we have gone over the interval.
-    const size_t gc_num = total_collections();
-    const size_t elapsed_gcs = gc_num - _evacuation_failure_alot_gc_number;
-
-    _evacuation_failure_alot_for_current_gc = (elapsed_gcs >= G1EvacuationFailureALotInterval);
-
-    // Now check if G1EvacuationFailureALot is enabled for the current GC type.
-    const bool in_young_only_phase = collector_state()->in_young_only_phase();
-    const bool in_concurrent_start_gc = collector_state()->in_concurrent_start_gc();
-    const bool mark_or_rebuild_in_progress = collector_state()->mark_or_rebuild_in_progress();
-
-    _evacuation_failure_alot_for_current_gc &=
-      evacuation_failure_alot_for_gc_type(in_young_only_phase,
-                                          in_concurrent_start_gc,
-                                          mark_or_rebuild_in_progress);
-  }
-}
-
-inline bool G1CollectedHeap::evacuation_should_fail() {
-  if (!G1EvacuationFailureALot || !_evacuation_failure_alot_for_current_gc) {
-    return false;
-  }
-  // G1EvacuationFailureALot is in effect for current GC
-  // Access to _evacuation_failure_alot_count is not atomic;
-  // the value does not have to be exact.
-  if (++_evacuation_failure_alot_count < G1EvacuationFailureALotCount) {
-    return false;
-  }
-  _evacuation_failure_alot_count = 0;
-  return true;
-}
-
-inline void G1CollectedHeap::reset_evacuation_should_fail() {
-  if (G1EvacuationFailureALot) {
-    _evacuation_failure_alot_gc_number = total_collections();
-    _evacuation_failure_alot_count = 0;
-    _evacuation_failure_alot_for_current_gc = false;
-  }
-}
-#endif  // #ifndef PRODUCT
-
 inline bool G1CollectedHeap::is_in_young(const oop obj) {
   if (obj == NULL) {
     return false;

src/hotspot/share/gc/g1/g1ParScanThreadState.cpp

@@ -31,6 +31,7 @@
 #include "gc/g1/g1RootClosures.hpp"
 #include "gc/g1/g1StringDedup.hpp"
 #include "gc/g1/g1Trace.hpp"
+#include "gc/g1/g1YoungGCEvacFailureInjector.inline.hpp"
 #include "gc/shared/partialArrayTaskStepper.inline.hpp"
 #include "gc/shared/preservedMarks.inline.hpp"
 #include "gc/shared/stringdedup/stringDedup.hpp"
@@ -460,15 +461,13 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
   assert(obj_ptr != NULL, "when we get here, allocation should have succeeded");
   assert(_g1h->is_in_reserved(obj_ptr), "Allocated memory should be in the heap");
 
-#ifndef PRODUCT
   // Should this evacuation fail?
-  if (_g1h->evacuation_should_fail()) {
+  if (_g1h->evac_failure_injector()->evacuation_should_fail()) {
     // Doing this after all the allocation attempts also tests the
     // undo_allocation() method too.
     undo_allocation(dest_attr, obj_ptr, word_sz, node_index);
     return handle_evacuation_failure_par(old, old_mark, word_sz);
   }
-#endif // !PRODUCT
 
   // We're going to allocate linearly, so might as well prefetch ahead.
   Prefetch::write(obj_ptr, PrefetchCopyIntervalInBytes);

src/hotspot/share/gc/g1/g1YoungGCEvacFailureInjector.cpp

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1YoungGCEvacFailureInjector.inline.hpp"
+#include "gc/g1/g1_globals.hpp"
+
+#ifndef PRODUCT
+
+bool G1YoungGCEvacFailureInjector::arm_if_needed_for_gc_type(bool for_young_gc,
+                                                             bool during_concurrent_start,
+                                                             bool mark_or_rebuild_in_progress) {
+  bool res = false;
+  if (mark_or_rebuild_in_progress) {
+    res |= G1EvacuationFailureALotDuringConcMark;
+  }
+  if (during_concurrent_start) {
+    res |= G1EvacuationFailureALotDuringConcurrentStart;
+  }
+  if (for_young_gc) {
+    res |= G1EvacuationFailureALotDuringYoungGC;
+  } else {
+    // GCs are mixed
+    res |= G1EvacuationFailureALotDuringMixedGC;
+  }
+  return res;
+}
+
+void G1YoungGCEvacFailureInjector::arm_if_needed() {
+  if (G1EvacuationFailureALot) {
+    G1CollectedHeap* g1h = G1CollectedHeap::heap();
+    // Check if we have gone over the interval.
+    const size_t gc_num = g1h->total_collections();
+    const size_t elapsed_gcs = gc_num - _last_collection_with_evacuation_failure;
+
+    _inject_evacuation_failure_for_current_gc = (elapsed_gcs >= G1EvacuationFailureALotInterval);
+
+    // Now check if evacuation failure injection should be enabled for the current GC.
+    G1CollectorState* collector_state = g1h->collector_state();
+    const bool in_young_only_phase = collector_state->in_young_only_phase();
+    const bool in_concurrent_start_gc = collector_state->in_concurrent_start_gc();
+    const bool mark_or_rebuild_in_progress = collector_state->mark_or_rebuild_in_progress();
+
+    _inject_evacuation_failure_for_current_gc &=
+      arm_if_needed_for_gc_type(in_young_only_phase,
+                                in_concurrent_start_gc,
+                                mark_or_rebuild_in_progress);
+  }
+}
+
+void G1YoungGCEvacFailureInjector::reset() {
+  if (G1EvacuationFailureALot) {
+    _last_collection_with_evacuation_failure = G1CollectedHeap::heap()->total_collections();
+    _evacuation_failure_object_count = 0;
+    _inject_evacuation_failure_for_current_gc = false;
+  }
+}
+
+#endif // #ifndef PRODUCT

src/hotspot/share/gc/g1/g1YoungGCEvacFailureInjector.hpp

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1YOUNGGCEVACUATIONFAILUREINJECTOR_HPP
+#define SHARE_GC_G1_G1YOUNGGCEVACUATIONFAILUREINJECTOR_HPP
+
+#include "memory/allStatic.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+// Support for injecting evacuation failures based on the G1EvacuationFailureALot*
+// flags. Analogous to PromotionFailureALot for the other collectors.
+//
+// Every G1EvacuationFailureALotInterval collections without evacuation failure
+// inbetween we "arm" the injector to induce evacuation failures after
+// G1EvacuationFailureALotCount successful evacuations.
+//
+// Available only in non-product builds.
+class G1YoungGCEvacFailureInjector {
+#ifndef PRODUCT
+  // Should we inject evacuation failures in the current GC.
+  bool _inject_evacuation_failure_for_current_gc;
+
+  // Records the number of the last collection when evacuation failure happened.
+  // Used to determine whether evacuation failure injection should be in effect
+  // for the current GC.
+  size_t _last_collection_with_evacuation_failure;
+
+  // The number of evacuations between induced failures.
+  volatile size_t _evacuation_failure_object_count;
+#endif
+
+  bool arm_if_needed_for_gc_type(bool for_young_gc,
+                                 bool during_concurrent_start,
+                                 bool mark_or_rebuild_in_progress) PRODUCT_RETURN_( return false; );
+public:
+
+  // Arm the evacuation failure injector if needed for the current
+  // GC (based upon the type of GC and which command line flags are set);
+  void arm_if_needed() PRODUCT_RETURN;
+
+  // Return true if it's time to cause an evacuation failure.
+  bool evacuation_should_fail() PRODUCT_RETURN_( return false; );
+
+  // Reset the evacuation failure injection counters. Should be called at
+  // the end of an evacuation pause in which an evacuation failure occurred.
+  void reset() PRODUCT_RETURN;
+};
+
+#endif /* SHARE_GC_G1_G1YOUNGGCEVACUATIONFAILUREINJECTOR_HPP */
+

src/hotspot/share/gc/g1/g1YoungGCEvacFailureInjector.inline.hpp

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1YOUNGGCEVACUATIONFAILUREINJECTOR_INLINE_HPP
+#define SHARE_GC_G1_G1YOUNGGCEVACUATIONFAILUREINJECTOR_INLINE_HPP
+
+#include "gc/g1/g1_globals.hpp"
+#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1YoungGCEvacFailureInjector.hpp"
+
+#ifndef PRODUCT
+
+inline bool G1YoungGCEvacFailureInjector::evacuation_should_fail() {
+  if (!G1EvacuationFailureALot || !_inject_evacuation_failure_for_current_gc) {
+    return false;
+  }
+  // Injecting evacuation failures is in effect for current GC
+  // Access to _evacuation_failure_alot_count is not atomic;
+  // the value does not have to be exact.
+  if (++_evacuation_failure_object_count < G1EvacuationFailureALotCount) {
+    return false;
+  }
+  _evacuation_failure_object_count = 0;
+  return true;
+}
+
+#endif  // #ifndef PRODUCT
+
+#endif /* SHARE_GC_G1_G1YOUNGGCEVACUATIONFAILUREINJECTOR_INLINE_HPP */
+