8263495: Gather liveness info in the mark phase of G1 full gc

Co-authored-by: Shoubing Ma <mashoubing1@huawei.com>
Reviewed-by: tschatzl, sjohanss, ayang

src/hotspot/share/gc/g1/g1ConcurrentMark.cpp

@@ -2840,7 +2840,7 @@ G1CMTask::G1CMTask(uint worker_id,
   _cm(cm),
   _next_mark_bitmap(NULL),
   _task_queue(task_queue),
-  _mark_stats_cache(mark_stats, RegionMarkStatsCacheSize),
+  _mark_stats_cache(mark_stats, G1RegionMarkStatsCache::RegionMarkStatsCacheSize),
   _calls(0),
   _time_target_ms(0.0),
   _start_time_ms(0.0),

src/hotspot/share/gc/g1/g1ConcurrentMark.hpp

@@ -611,10 +611,6 @@ private:
     init_hash_seed                = 17
   };
 
-  // Number of entries in the per-task stats entry. This seems enough to have a very
-  // low cache miss rate.
-  static const uint RegionMarkStatsCacheSize = 1024;
-
   G1CMObjArrayProcessor       _objArray_processor;
 
   uint                        _worker_id;

src/hotspot/share/gc/g1/g1FullCollector.cpp

@@ -36,6 +36,7 @@
 #include "gc/g1/g1FullGCScope.hpp"
 #include "gc/g1/g1OopClosures.hpp"
 #include "gc/g1/g1Policy.hpp"
+#include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
 #include "gc/g1/g1StringDedup.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
 #include "gc/shared/preservedMarks.hpp"
@@ -122,8 +123,14 @@ G1FullCollector::G1FullCollector(G1CollectedHeap* heap, bool explicit_gc, bool c
   _preserved_marks_set.init(_num_workers);
   _markers = NEW_C_HEAP_ARRAY(G1FullGCMarker*, _num_workers, mtGC);
   _compaction_points = NEW_C_HEAP_ARRAY(G1FullGCCompactionPoint*, _num_workers, mtGC);
+
+  _live_stats = NEW_C_HEAP_ARRAY(G1RegionMarkStats, _heap->max_regions(), mtGC);
+  for (uint j = 0; j < heap->max_regions(); j++) {
+    _live_stats[j].clear();
+  }
+
   for (uint i = 0; i < _num_workers; i++) {
-    _markers[i] = new G1FullGCMarker(this, i, _preserved_marks_set.get(i));
+    _markers[i] = new G1FullGCMarker(this, i, _preserved_marks_set.get(i), _live_stats);
     _compaction_points[i] = new G1FullGCCompactionPoint();
     _oop_queue_set.register_queue(i, marker(i)->oop_stack());
     _array_queue_set.register_queue(i, marker(i)->objarray_stack());
@@ -138,6 +145,7 @@ G1FullCollector::~G1FullCollector() {
   }
   FREE_C_HEAP_ARRAY(G1FullGCMarker*, _markers);
   FREE_C_HEAP_ARRAY(G1FullGCCompactionPoint*, _compaction_points);
+  FREE_C_HEAP_ARRAY(G1RegionMarkStats, _live_stats);
 }
 
 class PrepareRegionsClosure : public HeapRegionClosure {

src/hotspot/share/gc/g1/g1FullCollector.hpp

@@ -30,6 +30,7 @@
 #include "gc/g1/g1FullGCMarker.hpp"
 #include "gc/g1/g1FullGCOopClosures.hpp"
 #include "gc/g1/g1FullGCScope.hpp"
+#include "gc/g1/g1RegionMarkStatsCache.hpp"
 #include "gc/shared/preservedMarks.hpp"
 #include "gc/shared/referenceProcessor.hpp"
 #include "gc/shared/taskqueue.hpp"
@@ -67,6 +68,7 @@ class G1FullCollector : StackObj {
   G1FullGCCompactionPoint   _serial_compaction_point;
   G1IsAliveClosure          _is_alive;
   ReferenceProcessorIsAliveMutator _is_alive_mutator;
+  G1RegionMarkStats*        _live_stats;
 
   static uint calc_active_workers();
 

src/hotspot/share/gc/g1/g1FullGCMarkTask.cpp

@@ -60,6 +60,7 @@ void G1FullGCMarkTask::work(uint worker_id) {
 
   // Mark stack is populated, now process and drain it.
   marker->complete_marking(collector()->oop_queue_set(), collector()->array_queue_set(), &_terminator);
+  marker->flush_mark_stats_cache();
 
   // This is the point where the entire marking should have completed.
   assert(marker->oop_stack()->is_empty(), "Marking should have completed");

src/hotspot/share/gc/g1/g1FullGCMarker.cpp

@@ -32,7 +32,8 @@
 
 G1FullGCMarker::G1FullGCMarker(G1FullCollector* collector,
                                uint worker_id,
-                               PreservedMarks* preserved_stack) :
+                               PreservedMarks* preserved_stack,
+                               G1RegionMarkStats* mark_stats) :
     _collector(collector),
     _worker_id(worker_id),
     _bitmap(collector->mark_bitmap()),
@@ -42,7 +43,9 @@ G1FullGCMarker::G1FullGCMarker(G1FullCollector* collector,
     _mark_closure(worker_id, this, G1CollectedHeap::heap()->ref_processor_stw()),
     _verify_closure(VerifyOption_G1UseFullMarking),
     _stack_closure(this),
-    _cld_closure(mark_closure(), ClassLoaderData::_claim_strong) {
+    _cld_closure(mark_closure(), ClassLoaderData::_claim_strong),
+    _mark_stats_cache(mark_stats, G1RegionMarkStatsCache::RegionMarkStatsCacheSize) {
+  _mark_stats_cache.reset();
   _oop_stack.initialize();
   _objarray_stack.initialize();
 }
@@ -67,3 +70,7 @@ void G1FullGCMarker::complete_marking(OopQueueSet* oop_stacks,
     }
   } while (!is_empty() || !terminator->offer_termination());
 }
+
+void G1FullGCMarker::flush_mark_stats_cache() {
+  _mark_stats_cache.evict_all();
+}

src/hotspot/share/gc/g1/g1FullGCMarker.hpp

@@ -26,6 +26,7 @@
 #define SHARE_GC_G1_G1FULLGCMARKER_HPP
 
 #include "gc/g1/g1FullGCOopClosures.hpp"
+#include "gc/g1/g1RegionMarkStatsCache.hpp"
 #include "gc/shared/preservedMarks.hpp"
 #include "gc/shared/taskqueue.hpp"
 #include "memory/iterator.hpp"
@@ -63,6 +64,9 @@ class G1FullGCMarker : public CHeapObj<mtGC> {
   G1FollowStackClosure _stack_closure;
   CLDToOopClosure      _cld_closure;
 
+
+  G1RegionMarkStatsCache _mark_stats_cache;
+
   inline bool is_empty();
   inline bool pop_object(oop& obj);
   inline bool pop_objarray(ObjArrayTask& array);
@@ -74,7 +78,10 @@ class G1FullGCMarker : public CHeapObj<mtGC> {
   inline void follow_array(objArrayOop array);
   inline void follow_array_chunk(objArrayOop array, int index);
 public:
-  G1FullGCMarker(G1FullCollector* collector, uint worker_id, PreservedMarks* preserved_stack);
+  G1FullGCMarker(G1FullCollector* collector,
+                 uint worker_id,
+                 PreservedMarks* preserved_stack,
+                 G1RegionMarkStats* mark_stats);
   ~G1FullGCMarker();
 
   // Stack getters
@@ -96,6 +103,9 @@ public:
   CLDToOopClosure*      cld_closure()   { return &_cld_closure; }
   G1MarkAndPushClosure* mark_closure()  { return &_mark_closure; }
   G1FollowStackClosure* stack_closure() { return &_stack_closure; }
+
+  // Flush live bytes to regions
+  void flush_mark_stats_cache();
 };
 
 #endif // SHARE_GC_G1_G1FULLGCMARKER_HPP

src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp

@@ -32,6 +32,7 @@
 #include "gc/g1/g1FullCollector.inline.hpp"
 #include "gc/g1/g1FullGCMarker.hpp"
 #include "gc/g1/g1FullGCOopClosures.inline.hpp"
+#include "gc/g1/g1RegionMarkStatsCache.hpp"
 #include "gc/g1/g1StringDedup.hpp"
 #include "gc/g1/g1StringDedupQueue.hpp"
 #include "gc/shared/preservedMarks.inline.hpp"
@@ -65,6 +66,10 @@ inline bool G1FullGCMarker::mark_object(oop obj) {
       java_lang_String::is_instance_inlined(obj)) {
     G1StringDedup::enqueue_from_mark(obj, _worker_id);
   }
+
+  // Collect live words.
+  _mark_stats_cache.add_live_words(obj);
+
   return true;
 }
 

src/hotspot/share/gc/g1/g1RegionMarkStatsCache.cpp

@@ -23,6 +23,7 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
 #include "memory/allocation.inline.hpp"
 #include "utilities/powerOfTwo.hpp"
@@ -44,6 +45,11 @@ G1RegionMarkStatsCache::~G1RegionMarkStatsCache() {
   FREE_C_HEAP_ARRAY(G1RegionMarkStatsCacheEntry, _cache);
 }
 
+void G1RegionMarkStatsCache::add_live_words(oop obj) {
+  uint hr_index = G1CollectedHeap::heap()->addr_to_region(cast_from_oop<HeapWord*>(obj));
+  add_live_words(hr_index, (size_t) obj->size());
+}
+
 // Evict all remaining statistics, returning cache hits and misses.
 Pair<size_t, size_t> G1RegionMarkStatsCache::evict_all() {
   for (uint i = 0; i < _num_cache_entries; i++) {
@@ -52,12 +58,13 @@ Pair<size_t, size_t> G1RegionMarkStatsCache::evict_all() {
   return Pair<size_t,size_t>(_cache_hits, _cache_misses);
 }
 
-// Reset all cache entries to their default values.
 void G1RegionMarkStatsCache::reset() {
   _cache_hits = 0;
   _cache_misses = 0;
 
   for (uint i = 0; i < _num_cache_entries; i++) {
-    _cache[i].clear();
+    // Avoid the initial cache miss and eviction by setting the i'th's cache
+    // region_idx to the region_idx due to how the hash is calculated.
+    _cache[i].clear(i);
   }
 }

src/hotspot/share/gc/g1/g1RegionMarkStatsCache.hpp

@@ -26,6 +26,7 @@
 #define SHARE_GC_G1_G1REGIONMARKSTATSCACHE_HPP
 
 #include "memory/allocation.hpp"
+#include "oops/oop.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/pair.hpp"
@@ -69,14 +70,10 @@ private:
     uint _region_idx;
     G1RegionMarkStats _stats;
 
-    void clear() {
-      _region_idx = 0;
+    void clear(uint idx = 0) {
+      _region_idx = idx;
       _stats.clear();
     }
-
-    bool is_clear() const {
-      return _region_idx == 0 && _stats.is_clear();
-    }
   };
 
   // The actual cache and its number of entries.
@@ -98,10 +95,15 @@ private:
 
   G1RegionMarkStatsCacheEntry* find_for_add(uint region_idx);
 public:
+  // Number of entries in the per-task stats entry. This value seems enough
+  // to have a very low cache miss rate.
+  static const uint RegionMarkStatsCacheSize = 1024;
+
   G1RegionMarkStatsCache(G1RegionMarkStats* target, uint num_cache_entries);
 
   ~G1RegionMarkStatsCache();
 
+  void add_live_words(oop obj);
   void add_live_words(uint region_idx, size_t live_words) {
     G1RegionMarkStatsCacheEntry* const cur = find_for_add(region_idx);
     cur->_stats._live_words += live_words;
@@ -118,7 +120,8 @@ public:
   // Evict all remaining statistics, returning cache hits and misses.
   Pair<size_t, size_t> evict_all();
 
-  // Reset all cache entries to their default values.
+  // Reset liveness of all cache entries to their default values,
+  // initialize _region_idx to avoid initial cache miss.
   void reset();
 
   size_t hits() const { return _cache_hits; }