diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index 1f257560bcb..8bf068df0a8 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -335,7 +335,13 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { size_t garbage = region->garbage(); size_t live_bytes = region->get_live_data_bytes(); - live_data += live_bytes; + if (!region->was_promoted_in_place()) { + // As currently implemented, region->get_live_data_bytes() represents bytes concurrently marked. + // Expansion of the region by promotion during concurrent marking is above TAMS, and is not included + // as live-data at [start of] old marking. + live_data += live_bytes; + } + // else, regions that were promoted in place had 0 old live data at mark start if (region->is_regular() || region->is_regular_pinned()) { // Only place regular or pinned regions with live data into the candidate set. @@ -374,7 +380,6 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { } } - // TODO: subtract from live_data bytes promoted during concurrent GC. _old_generation->set_live_bytes_at_last_mark(live_data); // Unlike young, we are more interested in efficiently packing OLD-gen than in reclaiming garbage first. We sort by live-data. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp index de45877994c..c9b956f9c2f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp @@ -223,7 +223,6 @@ void ShenandoahGenerationalEvacuationTask::promote_in_place(ShenandoahHeapRegion // We do not need to scan above TAMS because restored top equals tams assert(obj_addr == tams, "Expect loop to terminate when obj_addr equals tams"); - { ShenandoahHeapLocker locker(_heap->lock()); @@ -251,6 +250,7 @@ void ShenandoahGenerationalEvacuationTask::promote_in_place(ShenandoahHeapRegion // Transfer this region from young to old, increasing promoted_reserve if available space exceeds plab_min_size() _heap->free_set()->add_promoted_in_place_region_to_old_collector(region); region->set_affiliation(OLD_GENERATION); + region->set_promoted_in_place(); } } @@ -289,6 +289,7 @@ void ShenandoahGenerationalEvacuationTask::promote_humongous(ShenandoahHeapRegio r->index(), p2i(r->bottom()), p2i(r->top())); // We mark the entire humongous object's range as dirty after loop terminates, so no need to dirty the range here r->set_affiliation(OLD_GENERATION); + r->set_promoted_in_place(); } ShenandoahFreeSet* freeset = _heap->free_set(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 2ed5614c698..cf0dc5476d0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -262,6 +262,7 @@ private: HeapWord* volatile _update_watermark; uint _age; + bool _promoted_in_place; CENSUS_NOISE(uint _youth;) // tracks epochs of retrograde ageing (rejuvenation) ShenandoahSharedFlag _recycling; // Used to indicate that the region is being recycled; see try_recycle*(). @@ -354,6 +355,15 @@ public: inline void save_top_before_promote(); inline HeapWord* get_top_before_promote() const { return _top_before_promoted; } + + inline void set_promoted_in_place() { + _promoted_in_place = true; + } + + // Returns true iff this region was promoted in place subsequent to the most recent start of concurrent old marking. + inline bool was_promoted_in_place() { + return _promoted_in_place; + } inline void restore_top_before_promote(); inline size_t garbage_before_padded_for_promote() const; @@ -379,7 +389,13 @@ public: inline void increase_live_data_gc_words(size_t s); inline bool has_live() const; + + // Represents the number of live bytes identified by most recent marking effort. Does not include the bytes + // above TAMS. inline size_t get_live_data_bytes() const; + + // Represents the number of live words identified by most recent marking effort. Does not include the words + // above TAMS. inline size_t get_live_data_words() const; inline size_t garbage() const; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index 69673eb7a60..b9304ee9daa 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -152,6 +152,7 @@ inline void ShenandoahHeapRegion::internal_increase_live_data(size_t s) { inline void ShenandoahHeapRegion::clear_live_data() { AtomicAccess::store(&_live_data, (size_t)0); + _promoted_in_place = false; } inline size_t ShenandoahHeapRegion::get_live_data_words() const {