diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index ff62e8aa976..421997e06d2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -210,6 +210,12 @@ void ShenandoahControlThread::run_service() { ResourceMark rm; LogStream ls(lt); heap->phase_timings()->print_cycle_on(&ls); +#ifdef NOT_PRODUCT + ShenandoahEvacuationTracker* evac_tracker = heap->evac_tracker(); + ShenandoahCycleStats evac_stats = evac_tracker->flush_cycle_to_global(); + evac_tracker->print_evacuations_on(&ls, &evac_stats.workers, + &evac_stats.mutators); +#endif } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp index b1d474fa78d..499e1342083 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp @@ -30,10 +30,22 @@ #include "runtime/thread.hpp" #include "runtime/threadSMR.inline.hpp" +ShenandoahEvacuationStats::ShenandoahEvacuations* ShenandoahEvacuationStats::get_category( + ShenandoahAffiliation from, + ShenandoahAffiliation to) { + if (from == YOUNG_GENERATION) { + if (to == YOUNG_GENERATION) { + return &_young; + } + assert(to == OLD_GENERATION, "If not evacuating to young, must be promotion to old"); + return &_promotion; + } + assert(from == OLD_GENERATION, "If not evacuating from young, then must be from old"); + return &_old; +} + ShenandoahEvacuationStats::ShenandoahEvacuationStats() - : _evacuations_completed(0), _bytes_completed(0), - _evacuations_attempted(0), _bytes_attempted(0), - _use_age_table(ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring), + : _use_age_table(ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring), _age_table(nullptr) { if (_use_age_table) { _age_table = new AgeTable(false); @@ -45,14 +57,17 @@ AgeTable* ShenandoahEvacuationStats::age_table() const { return _age_table; } -void ShenandoahEvacuationStats::begin_evacuation(size_t bytes) { - ++_evacuations_attempted; - _bytes_attempted += bytes; +void ShenandoahEvacuationStats::begin_evacuation(size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to) { + ShenandoahEvacuations* category = get_category(from, to); + category->_evacuations_attempted++; + category->_bytes_attempted += bytes; + } -void ShenandoahEvacuationStats::end_evacuation(size_t bytes) { - ++_evacuations_completed; - _bytes_completed += bytes; +void ShenandoahEvacuationStats::end_evacuation(size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to) { + ShenandoahEvacuations* category = get_category(from, to); + category->_evacuations_completed++; + category->_bytes_completed += bytes; } void ShenandoahEvacuationStats::record_age(size_t bytes, uint age) { @@ -63,34 +78,39 @@ void ShenandoahEvacuationStats::record_age(size_t bytes, uint age) { } void ShenandoahEvacuationStats::accumulate(const ShenandoahEvacuationStats* other) { - _evacuations_completed += other->_evacuations_completed; - _bytes_completed += other->_bytes_completed; - _evacuations_attempted += other->_evacuations_attempted; - _bytes_attempted += other->_bytes_attempted; + _young.accumulate(other->_young); + _old.accumulate(other->_old); + _promotion.accumulate(other->_promotion); + if (_use_age_table) { _age_table->merge(other->age_table()); } } void ShenandoahEvacuationStats::reset() { - _evacuations_completed = _evacuations_attempted = 0; - _bytes_completed = _bytes_attempted = 0; + _young.reset(); + _old.reset(); + _promotion.reset(); + if (_use_age_table) { _age_table->clear(); } } -void ShenandoahEvacuationStats::print_on(outputStream* st) { -#ifndef PRODUCT +void ShenandoahEvacuationStats::ShenandoahEvacuations::print_on(outputStream* st) const { size_t abandoned_size = _bytes_attempted - _bytes_completed; size_t abandoned_count = _evacuations_attempted - _evacuations_completed; - st->print_cr("Evacuated %zu%s across %zu objects, " - "abandoned %zu%s across %zu objects.", - byte_size_in_proper_unit(_bytes_completed), proper_unit_for_byte_size(_bytes_completed), - _evacuations_completed, - byte_size_in_proper_unit(abandoned_size), proper_unit_for_byte_size(abandoned_size), - abandoned_count); -#endif + st->print_cr("Evacuated " PROPERFMT" across %zu objects, " + "abandoned " PROPERFMT " across %zu objects.", + PROPERFMTARGS(_bytes_completed), _evacuations_completed, + PROPERFMTARGS(abandoned_size), abandoned_count); +} + +void ShenandoahEvacuationStats::print_on(outputStream* st) const { + st->print("Young: "); _young.print_on(st); + st->print("Promotion: "); _promotion.print_on(st); + st->print("Old: "); _old.print_on(st); + if (_use_age_table) { _age_table->print_on(st); } @@ -103,10 +123,10 @@ void ShenandoahEvacuationTracker::print_global_on(outputStream* st) { void ShenandoahEvacuationTracker::print_evacuations_on(outputStream* st, ShenandoahEvacuationStats* workers, ShenandoahEvacuationStats* mutators) { - st->print("Workers: "); + st->print_cr("Workers: "); workers->print_on(st); st->cr(); - st->print("Mutators: "); + st->print_cr("Mutators: "); mutators->print_on(st); st->cr(); @@ -160,12 +180,12 @@ ShenandoahCycleStats ShenandoahEvacuationTracker::flush_cycle_to_global() { return {workers, mutators}; } -void ShenandoahEvacuationTracker::begin_evacuation(Thread* thread, size_t bytes) { - ShenandoahThreadLocalData::begin_evacuation(thread, bytes); +void ShenandoahEvacuationTracker::begin_evacuation(Thread* thread, size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to) { + ShenandoahThreadLocalData::begin_evacuation(thread, bytes, from, to); } -void ShenandoahEvacuationTracker::end_evacuation(Thread* thread, size_t bytes) { - ShenandoahThreadLocalData::end_evacuation(thread, bytes); +void ShenandoahEvacuationTracker::end_evacuation(Thread* thread, size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to) { + ShenandoahThreadLocalData::end_evacuation(thread, bytes, from, to); } void ShenandoahEvacuationTracker::record_age(Thread* thread, size_t bytes, uint age) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp index 7d195656b11..e5d7a7fec94 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp @@ -26,14 +26,45 @@ #define SHARE_GC_SHENANDOAH_SHENANDOAHEVACTRACKER_HPP #include "gc/shared/ageTable.hpp" +#include "gc/shenandoah/shenandoahAffiliation.hpp" #include "utilities/ostream.hpp" class ShenandoahEvacuationStats : public CHeapObj { private: - size_t _evacuations_completed; - size_t _bytes_completed; - size_t _evacuations_attempted; - size_t _bytes_attempted; + struct ShenandoahEvacuations { + size_t _evacuations_completed; + size_t _bytes_completed; + size_t _evacuations_attempted; + size_t _bytes_attempted; + ShenandoahEvacuations() + : _evacuations_completed(0) + , _bytes_completed(0) + , _evacuations_attempted(0) + , _bytes_attempted(0) { + } + + void accumulate(const ShenandoahEvacuations& other) { + _evacuations_completed += other._evacuations_completed; + _bytes_completed += other._bytes_completed; + _evacuations_attempted += other._evacuations_attempted; + _bytes_attempted += other._bytes_attempted; + } + + void reset() { + _evacuations_completed = 0; + _bytes_completed = 0; + _evacuations_attempted = 0; + _bytes_attempted = 0; + } + + void print_on(outputStream* st) const; + }; + + ShenandoahEvacuations* get_category(ShenandoahAffiliation from, ShenandoahAffiliation to); + + ShenandoahEvacuations _young; + ShenandoahEvacuations _old; + ShenandoahEvacuations _promotion; bool _use_age_table; AgeTable* _age_table; @@ -43,11 +74,14 @@ private: AgeTable* age_table() const; - void begin_evacuation(size_t bytes); - void end_evacuation(size_t bytes); + // Record that the current thread is attempting to copy this many bytes. + void begin_evacuation(size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to); + + // Record that the current thread has completed copying this many bytes. + void end_evacuation(size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to); void record_age(size_t bytes, uint age); - void print_on(outputStream* st); + void print_on(outputStream* st) const; void accumulate(const ShenandoahEvacuationStats* other); void reset(); }; @@ -66,8 +100,12 @@ private: public: ShenandoahEvacuationTracker() = default; - void begin_evacuation(Thread* thread, size_t bytes); - void end_evacuation(Thread* thread, size_t bytes); + // Record that the given thread has begun to evacuate an object of this size. + void begin_evacuation(Thread* thread, size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to); + + // Multiple threads may attempt to evacuate the same object, but only the successful thread will end the evacuation. + // Evacuations that were begun, but not ended are considered 'abandoned'. + void end_evacuation(Thread* thread, size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to); void record_age(Thread* thread, size_t bytes, uint age); void print_global_on(outputStream* st); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp index a89fa76ba0f..d05ae713645 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp @@ -80,7 +80,6 @@ size_t ShenandoahGenerationalHeap::unsafe_max_tlab_alloc(Thread *thread) const { ShenandoahGenerationalHeap::ShenandoahGenerationalHeap(ShenandoahCollectorPolicy* policy) : ShenandoahHeap(policy), _age_census(nullptr), - _evac_tracker(new ShenandoahEvacuationTracker()), _min_plab_size(calculate_min_plab()), _max_plab_size(calculate_max_plab()), _regulator_thread(nullptr), @@ -100,18 +99,6 @@ void ShenandoahGenerationalHeap::print_init_logger() const { logger.print_all(); } -void ShenandoahGenerationalHeap::print_tracing_info() const { - ShenandoahHeap::print_tracing_info(); - - LogTarget(Info, gc, stats) lt; - if (lt.is_enabled()) { - LogStream ls(lt); - ls.cr(); - ls.cr(); - evac_tracker()->print_global_on(&ls); - } -} - void ShenandoahGenerationalHeap::initialize_heuristics() { // Initialize global generation and heuristics even in generational mode. ShenandoahHeap::initialize_heuristics(); @@ -338,7 +325,7 @@ oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, Shena } // Copy the object: - NOT_PRODUCT(evac_tracker()->begin_evacuation(thread, size * HeapWordSize)); + NOT_PRODUCT(evac_tracker()->begin_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen)); Copy::aligned_disjoint_words(cast_from_oop(p), copy, size); oop copy_val = cast_to_oop(copy); @@ -360,7 +347,7 @@ oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, Shena ContinuationGCSupport::relativize_stack_chunk(copy_val); // Record that the evacuation succeeded - NOT_PRODUCT(evac_tracker()->end_evacuation(thread, size * HeapWordSize)); + NOT_PRODUCT(evac_tracker()->end_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen)); if (target_gen == OLD_GENERATION) { old_generation()->handle_evacuation(copy, size, from_region->is_young()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp index 930c8ef7105..f23e49735e9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp @@ -36,7 +36,6 @@ class ShenandoahGenerationalControlThread; class ShenandoahAgeCensus; class ShenandoahGenerationalHeap : public ShenandoahHeap { - void print_tracing_info() const override; void stop() override; public: @@ -66,8 +65,6 @@ private: ShenandoahSharedFlag _is_aging_cycle; // Age census used for adapting tenuring threshold ShenandoahAgeCensus* _age_census; - // Used primarily to look for failed evacuation attempts. - ShenandoahEvacuationTracker* _evac_tracker; public: void set_aging_cycle(bool cond) { @@ -83,9 +80,6 @@ public: return _age_census; } - ShenandoahEvacuationTracker* evac_tracker() const { - return _evac_tracker; - } // Ages regions that haven't been used for allocations in the current cycle. // Resets ages for regions that have been used for allocations. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 2dc768363d1..72ee7a67e2a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -567,7 +567,8 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _bitmap_region_special(false), _aux_bitmap_region_special(false), _liveness_cache(nullptr), - _collection_set(nullptr) + _collection_set(nullptr), + _evac_tracker(new ShenandoahEvacuationTracker()) { // Initialize GC mode early, many subsequent initialization procedures depend on it initialize_mode(); @@ -1352,6 +1353,7 @@ oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapReg } // Copy the object: + NOT_PRODUCT(evac_tracker()->begin_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen)); Copy::aligned_disjoint_words(cast_from_oop(p), copy, size); // Try to install the new forwarding pointer. @@ -1361,6 +1363,7 @@ oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapReg // Successfully evacuated. Our copy is now the public one! ContinuationGCSupport::relativize_stack_chunk(copy_val); shenandoah_assert_correct(nullptr, copy_val); + NOT_PRODUCT(evac_tracker()->end_evacuation(thread, size * HeapWordSize, from_region->affiliation(), target_gen)); return copy_val; } else { // Failed to evacuate. We need to deal with the object that is left behind. Since this @@ -1590,6 +1593,13 @@ void ShenandoahHeap::print_tracing_info() const { ResourceMark rm; LogStream ls(lt); +#ifdef NOT_PRODUCT + evac_tracker()->print_global_on(&ls); + + ls.cr(); + ls.cr(); +#endif + phase_timings()->print_global_on(&ls); ls.cr(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index a32334019f1..4a9b9906863 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -557,6 +557,10 @@ public: ShenandoahEvacOOMHandler* oom_evac_handler() { return &_oom_evac_handler; } + ShenandoahEvacuationTracker* evac_tracker() const { + return _evac_tracker; + } + void on_cycle_start(GCCause::Cause cause, ShenandoahGeneration* generation); void on_cycle_end(ShenandoahGeneration* generation); @@ -789,6 +793,10 @@ private: oop try_evacuate_object(oop src, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahAffiliation target_gen); +protected: + // Used primarily to look for failed evacuation attempts. + ShenandoahEvacuationTracker* _evac_tracker; + public: static address in_cset_fast_test_addr(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp index c444a0ba86a..dd500462d0f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.cpp @@ -44,10 +44,7 @@ ShenandoahThreadLocalData::ShenandoahThreadLocalData() : _plab_promoted(0), _plab_allows_promotion(true), _plab_retries_enabled(true), - _evacuation_stats(nullptr) { - if (ShenandoahHeap::heap()->mode()->is_generational()) { - _evacuation_stats = new ShenandoahEvacuationStats(); - } + _evacuation_stats(new ShenandoahEvacuationStats()) { } ShenandoahThreadLocalData::~ShenandoahThreadLocalData() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp index c1cebdf1dde..098e20a72ec 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp @@ -30,6 +30,7 @@ #include "gc/shared/gcThreadLocalData.hpp" #include "gc/shared/plab.hpp" #include "gc/shenandoah/mode/shenandoahMode.hpp" +#include "gc/shenandoah/shenandoahAffiliation.hpp" #include "gc/shenandoah/shenandoahBarrierSet.hpp" #include "gc/shenandoah/shenandoahCardTable.hpp" #include "gc/shenandoah/shenandoahCodeRoots.hpp" @@ -159,12 +160,12 @@ public: data(thread)->_gclab_size = v; } - static void begin_evacuation(Thread* thread, size_t bytes) { - data(thread)->_evacuation_stats->begin_evacuation(bytes); + static void begin_evacuation(Thread* thread, size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to) { + data(thread)->_evacuation_stats->begin_evacuation(bytes, from, to); } - static void end_evacuation(Thread* thread, size_t bytes) { - data(thread)->_evacuation_stats->end_evacuation(bytes); + static void end_evacuation(Thread* thread, size_t bytes, ShenandoahAffiliation from, ShenandoahAffiliation to) { + data(thread)->_evacuation_stats->end_evacuation(bytes, from, to); } static void record_age(Thread* thread, size_t bytes, uint age) { @@ -172,7 +173,6 @@ public: } static ShenandoahEvacuationStats* evacuation_stats(Thread* thread) { - shenandoah_assert_generational(); return data(thread)->_evacuation_stats; }