8361726: Shenandoah: More detailed evacuation instrumentation

Reviewed-by: ysr, kdnilsen

src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp

@@ -210,6 +210,12 @@ void ShenandoahControlThread::run_service() {
           ResourceMark rm;
           LogStream ls(lt);
           heap->phase_timings()->print_cycle_on(&ls);
+#ifdef NOT_PRODUCT
+          ShenandoahEvacuationTracker* evac_tracker = heap->evac_tracker();
+          ShenandoahCycleStats         evac_stats   = evac_tracker->flush_cycle_to_global();
+          evac_tracker->print_evacuations_on(&ls, &evac_stats.workers,
+                                                  &evac_stats.mutators);
+#endif
         }
       }
 

src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp

@@ -30,10 +30,22 @@
 #include "runtime/thread.hpp"
 #include "runtime/threadSMR.inline.hpp"
 
+ShenandoahEvacuationStats::ShenandoahEvacuations* ShenandoahEvacuationStats::get_category(
+  ShenandoahAffiliation from,
+  ShenandoahAffiliation to) {
+  if (from == YOUNG_GENERATION) {
+    if (to == YOUNG_GENERATION) {
+      return &_young;
+    }
+    assert(to == OLD_GENERATION, "If not evacuating to young, must be promotion to old");
+    return &_promotion;
+  }
+  assert(from == OLD_GENERATION, "If not evacuating from young, then must be from old");
+  return &_old;
+}
+
 ShenandoahEvacuationStats::ShenandoahEvacuationStats()
-  : _evacuations_completed(0), _bytes_completed(0),
-    _evacuations_attempted(0), _bytes_attempted(0),
-    _use_age_table(ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring),
+  : _use_age_table(ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring),
     _age_table(nullptr) {
   if (_use_age_table) {
     _age_table = new AgeTable(false);
@@ -45,14 +57,17 @@ AgeTable* ShenandoahEvacuationStats::age_table() const {
   return _age_table;
 }
 
-void ShenandoahEvacuationStats::begin_evacuation(size_t bytes) {
-  ++_evacuations_attempted;
-  _bytes_attempted += bytes;
+void ShenandoahEvacuationStats::begin_evacuation(size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to) {
+  ShenandoahEvacuations* category = get_category(from, to);
+  category->_evacuations_attempted++;
+  category->_bytes_attempted += bytes;
+
 }
 
-void ShenandoahEvacuationStats::end_evacuation(size_t bytes) {
-  ++_evacuations_completed;
-  _bytes_completed += bytes;
+void ShenandoahEvacuationStats::end_evacuation(size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to) {
+  ShenandoahEvacuations* category = get_category(from, to);
+  category->_evacuations_completed++;
+  category->_bytes_completed += bytes;
 }
 
 void ShenandoahEvacuationStats::record_age(size_t bytes, uint age) {
@@ -63,34 +78,39 @@ void ShenandoahEvacuationStats::record_age(size_t bytes, uint age) {
 }
 
 void ShenandoahEvacuationStats::accumulate(const ShenandoahEvacuationStats* other) {
-  _evacuations_completed += other->_evacuations_completed;
-  _bytes_completed += other->_bytes_completed;
-  _evacuations_attempted += other->_evacuations_attempted;
-  _bytes_attempted += other->_bytes_attempted;
+  _young.accumulate(other->_young);
+  _old.accumulate(other->_old);
+  _promotion.accumulate(other->_promotion);
+
   if (_use_age_table) {
     _age_table->merge(other->age_table());
   }
 }
 
 void ShenandoahEvacuationStats::reset() {
-  _evacuations_completed = _evacuations_attempted = 0;
-  _bytes_completed = _bytes_attempted = 0;
+  _young.reset();
+  _old.reset();
+  _promotion.reset();
+
   if (_use_age_table) {
     _age_table->clear();
   }
 }
 
-void ShenandoahEvacuationStats::print_on(outputStream* st) {
-#ifndef PRODUCT
+void ShenandoahEvacuationStats::ShenandoahEvacuations::print_on(outputStream* st) const {
   size_t abandoned_size = _bytes_attempted - _bytes_completed;
   size_t abandoned_count = _evacuations_attempted - _evacuations_completed;
-  st->print_cr("Evacuated %zu%s across %zu objects, "
-            "abandoned %zu%s across %zu objects.",
-            byte_size_in_proper_unit(_bytes_completed), proper_unit_for_byte_size(_bytes_completed),
-            _evacuations_completed,
-            byte_size_in_proper_unit(abandoned_size),   proper_unit_for_byte_size(abandoned_size),
-            abandoned_count);
-#endif
+  st->print_cr("Evacuated " PROPERFMT" across %zu objects, "
+            "abandoned " PROPERFMT " across %zu objects.",
+            PROPERFMTARGS(_bytes_completed), _evacuations_completed,
+            PROPERFMTARGS(abandoned_size), abandoned_count);
+}
+
+void ShenandoahEvacuationStats::print_on(outputStream* st) const {
+  st->print("Young: "); _young.print_on(st);
+  st->print("Promotion: "); _promotion.print_on(st);
+  st->print("Old: "); _old.print_on(st);
+
   if (_use_age_table) {
     _age_table->print_on(st);
   }
@@ -103,10 +123,10 @@ void ShenandoahEvacuationTracker::print_global_on(outputStream* st) {
 void ShenandoahEvacuationTracker::print_evacuations_on(outputStream* st,
                                                        ShenandoahEvacuationStats* workers,
                                                        ShenandoahEvacuationStats* mutators) {
-  st->print("Workers: ");
+  st->print_cr("Workers: ");
   workers->print_on(st);
   st->cr();
-  st->print("Mutators: ");
+  st->print_cr("Mutators: ");
   mutators->print_on(st);
   st->cr();
 
@@ -160,12 +180,12 @@ ShenandoahCycleStats ShenandoahEvacuationTracker::flush_cycle_to_global() {
   return {workers, mutators};
 }
 
-void ShenandoahEvacuationTracker::begin_evacuation(Thread* thread, size_t bytes) {
-  ShenandoahThreadLocalData::begin_evacuation(thread, bytes);
+void ShenandoahEvacuationTracker::begin_evacuation(Thread* thread, size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to) {
+  ShenandoahThreadLocalData::begin_evacuation(thread, bytes, from, to);
 }
 
-void ShenandoahEvacuationTracker::end_evacuation(Thread* thread, size_t bytes) {
-  ShenandoahThreadLocalData::end_evacuation(thread, bytes);
+void ShenandoahEvacuationTracker::end_evacuation(Thread* thread, size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to) {
+  ShenandoahThreadLocalData::end_evacuation(thread, bytes, from, to);
 }
 
 void ShenandoahEvacuationTracker::record_age(Thread* thread, size_t bytes, uint age) {

src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp

@@ -26,14 +26,45 @@
 #define SHARE_GC_SHENANDOAH_SHENANDOAHEVACTRACKER_HPP
 
 #include "gc/shared/ageTable.hpp"
+#include "gc/shenandoah/shenandoahAffiliation.hpp"
 #include "utilities/ostream.hpp"
 
 class ShenandoahEvacuationStats : public CHeapObj<mtGC> {
 private:
-  size_t _evacuations_completed;
-  size_t _bytes_completed;
-  size_t _evacuations_attempted;
-  size_t _bytes_attempted;
+  struct ShenandoahEvacuations {
+    size_t _evacuations_completed;
+    size_t _bytes_completed;
+    size_t _evacuations_attempted;
+    size_t _bytes_attempted;
+    ShenandoahEvacuations()
+      : _evacuations_completed(0)
+      , _bytes_completed(0)
+      , _evacuations_attempted(0)
+      , _bytes_attempted(0) {
+    }
+
+    void accumulate(const ShenandoahEvacuations& other) {
+      _evacuations_completed += other._evacuations_completed;
+      _bytes_completed += other._bytes_completed;
+      _evacuations_attempted += other._evacuations_attempted;
+      _bytes_attempted += other._bytes_attempted;
+    }
+
+    void reset() {
+      _evacuations_completed = 0;
+      _bytes_completed = 0;
+      _evacuations_attempted = 0;
+      _bytes_attempted = 0;
+    }
+
+    void print_on(outputStream* st) const;
+  };
+
+  ShenandoahEvacuations* get_category(ShenandoahAffiliation from, ShenandoahAffiliation to);
+
+  ShenandoahEvacuations _young;
+  ShenandoahEvacuations _old;
+  ShenandoahEvacuations _promotion;
 
   bool      _use_age_table;
   AgeTable* _age_table;
@@ -43,11 +74,14 @@ private:
 
   AgeTable* age_table() const;
 
-  void begin_evacuation(size_t bytes);
-  void end_evacuation(size_t bytes);
+  // Record that the current thread is attempting to copy this many bytes.
+  void begin_evacuation(size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to);
+
+  // Record that the current thread has completed copying this many bytes.
+  void end_evacuation(size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to);
   void record_age(size_t bytes, uint age);
 
-  void print_on(outputStream* st);
+  void print_on(outputStream* st) const;
   void accumulate(const ShenandoahEvacuationStats* other);
   void reset();
 };
@@ -66,8 +100,12 @@ private:
 public:
   ShenandoahEvacuationTracker() = default;
 
-  void begin_evacuation(Thread* thread, size_t bytes);
-  void end_evacuation(Thread* thread, size_t bytes);
+  // Record that the given thread has begun to evacuate an object of this size.
+  void begin_evacuation(Thread* thread, size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to);
+
+  // Multiple threads may attempt to evacuate the same object, but only the successful thread will end the evacuation.
+  // Evacuations that were begun, but not ended are considered 'abandoned'.
+  void end_evacuation(Thread* thread, size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to);
   void record_age(Thread* thread, size_t bytes, uint age);
 
   void print_global_on(outputStream* st);

src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp

@@ -80,7 +80,6 @@ size_t ShenandoahGenerationalHeap::unsafe_max_tlab_alloc(Thread *thread) const {
 ShenandoahGenerationalHeap::ShenandoahGenerationalHeap(ShenandoahCollectorPolicy* policy) :
   ShenandoahHeap(policy),
   _age_census(nullptr),
-  _evac_tracker(new ShenandoahEvacuationTracker()),
   _min_plab_size(calculate_min_plab()),
   _max_plab_size(calculate_max_plab()),
   _regulator_thread(nullptr),
@@ -100,18 +99,6 @@ void ShenandoahGenerationalHeap::print_init_logger() const {
   logger.print_all();
 }
 
-void ShenandoahGenerationalHeap::print_tracing_info() const {
-  ShenandoahHeap::print_tracing_info();
-
-  LogTarget(Info, gc, stats) lt;
-  if (lt.is_enabled()) {
-    LogStream ls(lt);
-    ls.cr();
-    ls.cr();
-    evac_tracker()->print_global_on(&ls);
-  }
-}
-
 void ShenandoahGenerationalHeap::initialize_heuristics() {
   // Initialize global generation and heuristics even in generational mode.
   ShenandoahHeap::initialize_heuristics();
@@ -338,7 +325,7 @@ oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, Shena
   }
 
   // Copy the object:
-  NOT_PRODUCT(evac_tracker()->begin_evacuation(thread, size * HeapWordSize));
+  NOT_PRODUCT(evac_tracker()->begin_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen));
   Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
   oop copy_val = cast_to_oop(copy);
 
@@ -360,7 +347,7 @@ oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, Shena
     ContinuationGCSupport::relativize_stack_chunk(copy_val);
 
     // Record that the evacuation succeeded
-    NOT_PRODUCT(evac_tracker()->end_evacuation(thread, size * HeapWordSize));
+    NOT_PRODUCT(evac_tracker()->end_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen));
 
     if (target_gen == OLD_GENERATION) {
       old_generation()->handle_evacuation(copy, size, from_region->is_young());

src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp

@@ -36,7 +36,6 @@ class ShenandoahGenerationalControlThread;
 class ShenandoahAgeCensus;
 
 class ShenandoahGenerationalHeap : public ShenandoahHeap {
-  void print_tracing_info() const override;
   void stop() override;
 
 public:
@@ -66,8 +65,6 @@ private:
   ShenandoahSharedFlag  _is_aging_cycle;
   // Age census used for adapting tenuring threshold
   ShenandoahAgeCensus* _age_census;
-  // Used primarily to look for failed evacuation attempts.
-  ShenandoahEvacuationTracker*  _evac_tracker;
 
 public:
   void set_aging_cycle(bool cond) {
@@ -83,9 +80,6 @@ public:
     return _age_census;
   }
 
-  ShenandoahEvacuationTracker* evac_tracker() const {
-    return _evac_tracker;
-  }
 
   // Ages regions that haven't been used for allocations in the current cycle.
   // Resets ages for regions that have been used for allocations.

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -567,7 +567,8 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
   _bitmap_region_special(false),
   _aux_bitmap_region_special(false),
   _liveness_cache(nullptr),
-  _collection_set(nullptr)
+  _collection_set(nullptr),
+  _evac_tracker(new ShenandoahEvacuationTracker())
 {
   // Initialize GC mode early, many subsequent initialization procedures depend on it
   initialize_mode();
@@ -1352,6 +1353,7 @@ oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapReg
   }
 
   // Copy the object:
+  NOT_PRODUCT(evac_tracker()->begin_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen));
   Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
 
   // Try to install the new forwarding pointer.
@@ -1361,6 +1363,7 @@ oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapReg
     // Successfully evacuated. Our copy is now the public one!
     ContinuationGCSupport::relativize_stack_chunk(copy_val);
     shenandoah_assert_correct(nullptr, copy_val);
+    NOT_PRODUCT(evac_tracker()->end_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen));
     return copy_val;
   }  else {
     // Failed to evacuate. We need to deal with the object that is left behind. Since this
@@ -1590,6 +1593,13 @@ void ShenandoahHeap::print_tracing_info() const {
     ResourceMark rm;
     LogStream ls(lt);
 
+#ifdef NOT_PRODUCT
+    evac_tracker()->print_global_on(&ls);
+
+    ls.cr();
+    ls.cr();
+#endif
+
     phase_timings()->print_global_on(&ls);
 
     ls.cr();

src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp

@@ -557,6 +557,10 @@ public:
 
   ShenandoahEvacOOMHandler*  oom_evac_handler()        { return &_oom_evac_handler; }
 
+  ShenandoahEvacuationTracker* evac_tracker() const {
+    return _evac_tracker;
+  }
+
   void on_cycle_start(GCCause::Cause cause, ShenandoahGeneration* generation);
   void on_cycle_end(ShenandoahGeneration* generation);
 
@@ -789,6 +793,10 @@ private:
 
   oop try_evacuate_object(oop src, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahAffiliation target_gen);
 
+protected:
+  // Used primarily to look for failed evacuation attempts.
+  ShenandoahEvacuationTracker*  _evac_tracker;
+
 public:
   static address in_cset_fast_test_addr();
 

src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp

@@ -44,10 +44,7 @@ ShenandoahThreadLocalData::ShenandoahThreadLocalData() :
   _plab_promoted(0),
   _plab_allows_promotion(true),
   _plab_retries_enabled(true),
-  _evacuation_stats(nullptr) {
-  if (ShenandoahHeap::heap()->mode()->is_generational()) {
-    _evacuation_stats = new ShenandoahEvacuationStats();
-  }
+  _evacuation_stats(new ShenandoahEvacuationStats()) {
 }
 
 ShenandoahThreadLocalData::~ShenandoahThreadLocalData() {

src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp

@@ -30,6 +30,7 @@
 #include "gc/shared/gcThreadLocalData.hpp"
 #include "gc/shared/plab.hpp"
 #include "gc/shenandoah/mode/shenandoahMode.hpp"
+#include "gc/shenandoah/shenandoahAffiliation.hpp"
 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc/shenandoah/shenandoahCardTable.hpp"
 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
@@ -159,12 +160,12 @@ public:
     data(thread)->_gclab_size = v;
   }
 
-  static void begin_evacuation(Thread* thread, size_t bytes) {
-    data(thread)->_evacuation_stats->begin_evacuation(bytes);
+  static void begin_evacuation(Thread* thread, size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to) {
+    data(thread)->_evacuation_stats->begin_evacuation(bytes, from, to);
   }
 
-  static void end_evacuation(Thread* thread, size_t bytes) {
-    data(thread)->_evacuation_stats->end_evacuation(bytes);
+  static void end_evacuation(Thread* thread, size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to) {
+    data(thread)->_evacuation_stats->end_evacuation(bytes, from, to);
   }
 
   static void record_age(Thread* thread, size_t bytes, uint age) {
@@ -172,7 +173,6 @@ public:
   }
 
   static ShenandoahEvacuationStats* evacuation_stats(Thread* thread) {
-    shenandoah_assert_generational();
     return data(thread)->_evacuation_stats;
   }