From 533a8a8d2605f19f5f6f867c3ff5d649305d0ee5 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 15 Apr 2026 21:54:39 +0000 Subject: [PATCH] 8376839: GenShen: Improve performance of evacuations into the old generation Reviewed-by: kdnilsen --- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 23 +++ .../gc/shenandoah/shenandoahConcurrentGC.hpp | 4 +- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 5 + .../share/gc/shenandoah/shenandoahFreeSet.cpp | 3 +- .../shenandoahGenerationalEvacuationTask.cpp | 1 + .../shenandoah/shenandoahGenerationalHeap.cpp | 4 - .../share/gc/shenandoah/shenandoahHeap.cpp | 25 +++ .../share/gc/shenandoah/shenandoahHeap.hpp | 1 + .../gc/shenandoah/shenandoahHeapRegion.cpp | 6 + .../gc/shenandoah/shenandoahHeapRegion.hpp | 6 +- .../shenandoahHeapRegionClosures.cpp | 5 + .../shenandoah/shenandoahInPlacePromoter.cpp | 3 + .../gc/shenandoah/shenandoahOldGeneration.cpp | 33 +++- .../gc/shenandoah/shenandoahOldGeneration.hpp | 37 +++- .../share/gc/shenandoah/shenandoahPLAB.cpp | 8 - .../gc/shenandoah/shenandoahPhaseTimings.hpp | 3 +- .../shenandoah/shenandoahScanRemembered.cpp | 165 +++++++++++------- .../shenandoah/shenandoahScanRemembered.hpp | 7 + 18 files changed, 248 insertions(+), 91 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index f0125c38cae..ba0aa1a1304 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -112,6 +112,24 @@ void ShenandoahConcurrentGC::entry_concurrent_update_refs_prepare(ShenandoahHeap heap->concurrent_prepare_for_update_refs(); } +void ShenandoahConcurrentGC::entry_update_card_table() { + ShenandoahHeap* const heap = ShenandoahHeap::heap(); + TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); + + static const char* msg = "Concurrent update cards"; + ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_card_table); + EventMark em("%s", msg); + + ShenandoahWorkerScope scope(heap->workers(), + ShenandoahWorkerPolicy::calc_workers_for_conc_evac(), + "concurrent update cards"); + + // Heap needs to be parsable here. + // Also, parallel heap region iterate must have a phase set. + assert(ShenandoahTimingsTracker::is_current_phase_valid(), "Current phase must be set"); + ShenandoahGenerationalHeap::heap()->old_generation()->update_card_table(); +} + bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { ShenandoahHeap* const heap = ShenandoahHeap::heap(); _generation->ref_processor()->set_soft_reference_policy( @@ -206,6 +224,11 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { // Perform update-refs phase. entry_concurrent_update_refs_prepare(heap); + + if (ShenandoahHeap::heap()->mode()->is_generational()) { + entry_update_card_table(); + } + if (ShenandoahVerify) { vmop_entry_init_update_refs(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp index 54d43416fdb..ba228901d72 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp @@ -59,8 +59,6 @@ public: bool collect(GCCause::Cause cause) override; ShenandoahDegenPoint degen_point() const; - void entry_concurrent_update_refs_prepare(ShenandoahHeap* heap); - // Return true if this cycle found enough immediate garbage to skip evacuation bool abbreviated() const { return _abbreviated; } @@ -95,6 +93,8 @@ protected: void entry_cleanup_early(); void entry_evacuate(); void entry_update_thread_roots(); + void entry_update_card_table(); + void entry_concurrent_update_refs_prepare(ShenandoahHeap* heap); void entry_update_refs(); void entry_cleanup_complete(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 1873d818093..84b22f13d47 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -277,6 +277,11 @@ void ShenandoahDegenGC::op_degenerated() { _abbreviated = true; } + // labs are retired, walk the old regions and update remembered set + if (ShenandoahHeap::heap()->mode()->is_generational()) { + ShenandoahGenerationalHeap::heap()->old_generation()->update_card_table(); + } + case _degenerated_update_refs: if (heap->has_forwarded_objects()) { op_update_refs(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index a579d6d3694..592c5bffa5a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -1569,7 +1569,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // We must call try_recycle_under_lock() even if !r->is_trash(). The reason is that if r is being recycled at this // moment by a GC worker thread, it may appear to be not trash even though it has not yet been fully recycled. If // we proceed without waiting for the worker to finish recycling the region, the worker thread may overwrite the - // region's affiliation with FREE after we set the region's affiliation to req.afiliation() below + // region's affiliation with FREE after we set the region's affiliation to req.affiliation() below r->try_recycle_under_lock(); in_new_region = r->is_empty(); if (in_new_region) { @@ -1585,7 +1585,6 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // concurrent preparations for mixed evacuations are completed), we mark this region as not requiring any // coalesce-and-fill processing. r->end_preemptible_coalesce_and_fill(); - _heap->old_generation()->clear_cards_for(r); } #ifdef ASSERT ShenandoahMarkingContext* const ctx = _heap->marking_context(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp index 6912750378e..ca15c6db443 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp @@ -133,3 +133,4 @@ void ShenandoahGenerationalEvacuationTask::evacuate_and_promote_regions() { } } } + diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp index d5cfa4b7fb9..a51449e91f4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp @@ -363,10 +363,6 @@ oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, uint // Record that the evacuation succeeded evac_tracker()->end_evacuation(thread, size * HeapWordSize, FROM_GENERATION, TO_GENERATION); } - - if (TO_GENERATION == OLD_GENERATION) { - old_generation()->handle_evacuation(copy, size); - } } else { // Failed to evacuate. We need to deal with the object that is left behind. Since this // new allocation is certainly after TAMS, it will be considered live in the next cycle. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 5bf76505506..75d3ade4e49 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1955,6 +1955,26 @@ void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const } } +class ShenandoahHeapRegionIteratorTask : public WorkerTask { +private: + ShenandoahRegionIterator _regions; + ShenandoahHeapRegionClosure* _closure; + +public: + ShenandoahHeapRegionIteratorTask(ShenandoahHeapRegionClosure* closure) + : WorkerTask("Shenandoah Heap Region Iterator") + , _closure(closure) {} + + void work(uint worker_id) override { + ShenandoahParallelWorkerSession worker_session(worker_id); + ShenandoahHeapRegion* region = _regions.next(); + while (region != nullptr) { + _closure->heap_region_do(region); + region = _regions.next(); + } + } +}; + class ShenandoahParallelHeapRegionTask : public WorkerTask { private: ShenandoahHeap* const _heap; @@ -2011,6 +2031,11 @@ void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* b } } +void ShenandoahHeap::heap_region_iterator(ShenandoahHeapRegionClosure* closure) const { + ShenandoahHeapRegionIteratorTask task(closure); + workers()->run_task(&task); +} + class ShenandoahRendezvousHandshakeClosure : public HandshakeClosure { public: inline ShenandoahRendezvousHandshakeClosure(const char* name) : HandshakeClosure(name) {} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index d4604be0aec..ab7dd00b774 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -298,6 +298,7 @@ public: void heap_region_iterate(ShenandoahHeapRegionClosure* blk) const; void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const; + void heap_region_iterator(ShenandoahHeapRegionClosure* blk) const; inline ShenandoahMmuTracker* mmu_tracker() { return &_mmu_tracker; }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index afc6b24e168..3db11000af5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -67,6 +67,7 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool c _new_top(nullptr), _empty_time(os::elapsedTime()), _top_before_promoted(nullptr), + _top_at_evac_start(start), _state(committed ? _empty_committed : _empty_uncommitted), _top(start), _tlab_allocs(0), @@ -565,12 +566,17 @@ void ShenandoahHeapRegion::recycle_internal() { assert(_recycling.is_set() && is_trash(), "Wrong state"); ShenandoahHeap* heap = ShenandoahHeap::heap(); + _top_at_evac_start = _bottom; _mixed_candidate_garbage_words = 0; set_top(bottom()); clear_live_data(); reset_alloc_metadata(); heap->marking_context()->reset_top_at_mark_start(this); set_update_watermark(bottom()); + if (is_old()) { + heap->old_generation()->clear_cards_for(this); + } + if (ZapUnusedHeapArea) { SpaceMangler::mangle_region(MemRegion(bottom(), end())); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 3a0ac042f57..569b64f756b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -246,6 +246,7 @@ private: double _empty_time; HeapWord* _top_before_promoted; + HeapWord* _top_at_evac_start; // Seldom updated fields Atomic _state; @@ -365,12 +366,15 @@ public: } // Returns true iff this region was promoted in place subsequent to the most recent start of concurrent old marking. - inline bool was_promoted_in_place() { + bool was_promoted_in_place() const { return _promoted_in_place; } inline void restore_top_before_promote(); inline size_t garbage_before_padded_for_promote() const; + HeapWord* get_top_at_evac_start() const { return _top_at_evac_start; } + void record_top_at_evac_start() { _top_at_evac_start = _top; } + // If next available memory is not aligned on address that is multiple of alignment, fill the empty space // so that returned object is aligned on an address that is a multiple of alignment_in_bytes. Requested // size is in words. It is assumed that this->is_old(). A pad object is allocated, filled, and registered diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionClosures.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionClosures.cpp index 3c6fe1a3df1..7554a9c9a2c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionClosures.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionClosures.cpp @@ -80,6 +80,11 @@ void ShenandoahFinalMarkUpdateRegionStateClosure::heap_region_do(ShenandoahHeapR // Remember limit for updating refs. It's guaranteed that we get no // from-space-refs written from here on. r->set_update_watermark_at_safepoint(r->top()); + + if (r->is_old()) { + // Record where we need to start updating the remembered set + r->record_top_at_evac_start(); + } } else { assert(!r->has_live(), "Region %zu should have no live data", r->index()); assert(_ctx == nullptr || _ctx->top_at_mark_start(r) == r->top(), diff --git a/src/hotspot/share/gc/shenandoah/shenandoahInPlacePromoter.cpp b/src/hotspot/share/gc/shenandoah/shenandoahInPlacePromoter.cpp index 10d61221e87..153193fa3a3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahInPlacePromoter.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahInPlacePromoter.cpp @@ -238,6 +238,9 @@ void ShenandoahInPlacePromoter::promote(ShenandoahHeapRegion* region) const { // is_collector_free range. We'll add it to that range below. region->restore_top_before_promote(); + // We also need to record where those allocations begin so that we can later update the remembered set. + region->record_top_at_evac_start(); + assert(region->used() + pip_pad_bytes + pip_unpadded == region_size_bytes, "invariant"); // The update_watermark was likely established while we had the artificially high value of top. Make it sane now. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 4ad7d2a1ae5..37de5966554 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -318,6 +318,11 @@ void ShenandoahOldGeneration::heap_region_iterate(ShenandoahHeapRegionClosure* c ShenandoahHeap::heap()->heap_region_iterate(&old_regions_cl); } +void ShenandoahOldGeneration::heap_region_iterator(ShenandoahHeapRegionClosure* cl) { + ShenandoahIncludeRegionClosure old_regions_cl(cl); + ShenandoahHeap::heap()->heap_region_iterator(&old_regions_cl); +} + void ShenandoahOldGeneration::set_concurrent_mark_in_progress(bool in_progress) { ShenandoahHeap::heap()->set_concurrent_old_mark_in_progress(in_progress); } @@ -326,6 +331,12 @@ bool ShenandoahOldGeneration::is_concurrent_mark_in_progress() { return ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress(); } +void ShenandoahOldGeneration::record_tops_at_evac_start() { + for_each_region([](ShenandoahHeapRegion* region) { + region->record_top_at_evac_start(); + }); +} + void ShenandoahOldGeneration::cancel_marking() { if (is_concurrent_mark_in_progress()) { log_debug(gc)("Abandon SATB buffers"); @@ -662,15 +673,19 @@ void ShenandoahOldGeneration::log_failed_promotion(LogStream& ls, Thread* thread } } -void ShenandoahOldGeneration::handle_evacuation(HeapWord* obj, size_t words) const { - // Only register the copy of the object that won the evacuation race. - _card_scan->register_object_without_lock(obj); - - // Mark the entire range of the evacuated object as dirty. At next remembered set scan, - // we will clear dirty bits that do not hold interesting pointers. It's more efficient to - // do this in batch, in a background GC thread than to try to carefully dirty only cards - // that hold interesting pointers right now. - _card_scan->mark_range_as_dirty(obj, words); +void ShenandoahOldGeneration::update_card_table() { + for_each_region([this](ShenandoahHeapRegion* region) { + if (region->is_regular()) { + // Humongous regions are promoted in place, remembered set maintenance is handled there + // Regular regions that are promoted in place have their rset maintenance handled for + // the objects in the region when it was promoted. We record TEAS for such a region + // when the in-place-promotion is completed. Such a region may be used for additional + // promotions in the same cycle it was itself promoted. + if (region->top() > region->get_top_at_evac_start()) { + _card_scan->update_card_table(region->get_top_at_evac_start(), region->top()); + } + } + }); } bool ShenandoahOldGeneration::has_unprocessed_collection_candidates() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp index 0069d38a84e..942f93c5c68 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp @@ -172,8 +172,8 @@ public: void handle_failed_promotion(Thread* thread, size_t size) const; void log_failed_promotion(LogStream& ls, Thread* thread, size_t size) const; - // A successful evacuation re-dirties the cards and registers the object with the remembered set - void handle_evacuation(HeapWord* obj, size_t words) const; + // Iterate over recently promoted objects to update card table and object registrations + void update_card_table(); // Clear the flag after it is consumed by the control thread bool clear_failed_evacuation() { @@ -199,11 +199,36 @@ public: // Mark card for this location as dirty void mark_card_as_dirty(void* location); + template + class ShenandoahHeapRegionLambda : public ShenandoahHeapRegionClosure { + T _region_lambda; + public: + explicit ShenandoahHeapRegionLambda(T region_lambda) : _region_lambda(region_lambda) {} + + void heap_region_do(ShenandoahHeapRegion* r) override { + _region_lambda(r); + } + + bool is_thread_safe() override { + return true; + } + + size_t parallel_region_stride() override { + // Temporarily override to force parallelism when updating card table + return 8; + } + }; + + template + void for_each_region(LambdaT lambda) { + ShenandoahHeapRegionLambda l(lambda); + heap_region_iterator(&l); + } + void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; - void parallel_heap_region_iterate_free(ShenandoahHeapRegionClosure* cl) override; - void heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; + void heap_region_iterator(ShenandoahHeapRegionClosure* cl); bool contains(ShenandoahAffiliation affiliation) const override; bool contains(ShenandoahHeapRegion* region) const override; @@ -212,6 +237,10 @@ public: void set_concurrent_mark_in_progress(bool in_progress) override; bool is_concurrent_mark_in_progress() override; + // For old regions, objects between top at evac start and top represent promoted objects. + // These objects will need to have their cards dirtied and their offsets within the cards registered. + void record_tops_at_evac_start(); + bool entry_coalesce_and_fill(); // Global collections touch old regions, so the old generation needs to be informed of this. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPLAB.cpp b/src/hotspot/share/gc/shenandoah/shenandoahPLAB.cpp index 412cfa9447e..5049113b665 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPLAB.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPLAB.cpp @@ -210,12 +210,4 @@ void ShenandoahPLAB::retire() { // plab->retire() overwrites unused memory between plab->top() and plab->hard_end() with a dummy object to make memory parsable. // It adds the size of this unused memory, in words, to plab->waste(). _plab->retire(); - if (top != nullptr && _plab->waste() > original_waste && _heap->is_in_old(top)) { - // If retiring the plab created a filler object, then we need to register it with our card scanner so it can - // safely walk the region backing the plab. - log_debug(gc, plab)("retire_plab() is registering remnant of size %zu at " PTR_FORMAT, - (_plab->waste() - original_waste) * HeapWordSize, p2i(top)); - // No lock is necessary because the PLAB memory is aligned on card boundaries. - _heap->old_generation()->card_scan()->register_object_without_lock(top); - } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp index e890008b916..a454de68f00 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp @@ -109,6 +109,7 @@ class outputStream; f(conc_strong_roots, "Concurrent Strong Roots") \ SHENANDOAH_PAR_PHASE_DO(conc_strong_roots_, " CSR: ", f) \ f(conc_evac, "Concurrent Evacuation") \ + f(conc_update_card_table, "Concurrent Update Cards") \ f(conc_final_roots, "Concurrent Final Roots") \ f(promote_in_place, " Promote Regions") \ f(final_roots_gross, "Pause Verify Final Roots (G)") \ @@ -254,7 +255,7 @@ public: void flush_cycle_to_global(); static const char* phase_name(Phase phase) { - assert(phase >= 0 && phase < _num_phases, "Out of bound"); + assert(phase >= 0 && phase < _num_phases, "Out of bounds: %d", phase); return _phase_names[phase]; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp index 9e160d5b294..8d7ba2dc46f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -31,6 +31,33 @@ #include "logging/log.hpp" #include "runtime/threads.hpp" +// A closure that takes an oop in the old generation and, if it's pointing +// into the young generation, dirties the corresponding remembered set entry. +class ShenandoahDirtyRememberedSetClosure : public BasicOopIterateClosure { +protected: + ShenandoahGenerationalHeap* const _heap; + ShenandoahScanRemembered* const _scanner; + +public: + ShenandoahDirtyRememberedSetClosure() : + _heap(ShenandoahGenerationalHeap::heap()), + _scanner(_heap->old_generation()->card_scan()) {} + + template + void work(T* p) { + assert(_heap->is_in_old(p), "Expecting to get an old gen address"); + if (T o = RawAccess<>::oop_load(p); !CompressedOops::is_null(o)) { + if (const oop obj = CompressedOops::decode_not_null(o); _heap->is_in_young(obj)) { + // Dirty the card containing the cross-generational pointer. + _scanner->mark_card_as_dirty((HeapWord*) p); + } + } + } + + void do_oop(narrowOop* p) override { work(p); } + void do_oop(oop* p) override { work(p); } +}; + size_t ShenandoahDirectCardMarkRememberedSet::last_valid_index() const { return _card_table->last_valid_index(); } @@ -161,7 +188,6 @@ void ShenandoahCardCluster::register_object_without_lock(HeapWord* address) { uint8_t offset_in_card = checked_cast(pointer_delta(address, card_start_address)); if (!starts_object(card_at_start)) { - set_starts_object_bit(card_at_start); set_first_start(card_at_start, offset_in_card); set_last_start(card_at_start, offset_in_card); } else { @@ -172,6 +198,49 @@ void ShenandoahCardCluster::register_object_without_lock(HeapWord* address) { } } +void ShenandoahCardCluster::update_card_table(HeapWord* start, HeapWord* end) { + HeapWord* address = start; + HeapWord* previous_address = nullptr; + uint8_t previous_offset = 0; + size_t previous_card_index = -1; + ShenandoahDirtyRememberedSetClosure make_cards_dirty; + + log_debug(gc, remset)("Update remembered set from " PTR_FORMAT ", to " PTR_FORMAT, p2i(start), p2i(end)); + _rs->mark_range_as_dirty(start, pointer_delta(end, start)); + + while (address < end) { + + // Compute card and offset in card for this object + const size_t object_card_index = _rs->card_index_for_addr(address); + const HeapWord* card_start_address = _rs->addr_for_card_index(object_card_index); + const uint8_t offset_in_card = checked_cast(pointer_delta(address, card_start_address)); + + if (object_card_index != previous_card_index) { + if (previous_address != nullptr) { + // Register the previous object on the previous card, we are starting a new card here + set_last_start(previous_card_index, previous_offset); + } + + previous_card_index = object_card_index; + if (!starts_object(object_card_index)) { + // The previous cycle may have recorded an earlier start in this card. Do not overwrite it. + set_first_start(object_card_index, offset_in_card); + } + } + + previous_offset = offset_in_card; + previous_address = address; + + const oop obj = cast_to_oop(address); + address += obj->size(); + } + + // Register the last object seen in this range. + if (previous_address != nullptr) { + set_last_start(previous_card_index, previous_offset); + } +} + void ShenandoahCardCluster::coalesce_objects(HeapWord* address, size_t length_in_words) { size_t card_at_start = _rs->card_index_for_addr(address); @@ -641,36 +710,6 @@ void ShenandoahScanRemembered::merge_worker_card_stats_cumulative( } #endif -// A closure that takes an oop in the old generation and, if it's pointing -// into the young generation, dirties the corresponding remembered set entry. -// This is only used to rebuild the remembered set after a full GC. -class ShenandoahDirtyRememberedSetClosure : public BasicOopIterateClosure { -protected: - ShenandoahGenerationalHeap* const _heap; - ShenandoahScanRemembered* const _scanner; - -public: - ShenandoahDirtyRememberedSetClosure() : - _heap(ShenandoahGenerationalHeap::heap()), - _scanner(_heap->old_generation()->card_scan()) {} - - template - inline void work(T* p) { - assert(_heap->is_in_old(p), "Expecting to get an old gen address"); - T o = RawAccess<>::oop_load(p); - if (!CompressedOops::is_null(o)) { - oop obj = CompressedOops::decode_not_null(o); - if (_heap->is_in_young(obj)) { - // Dirty the card containing the cross-generational pointer. - _scanner->mark_card_as_dirty((HeapWord*) p); - } - } - } - - virtual void do_oop(narrowOop* p) { work(p); } - virtual void do_oop(oop* p) { work(p); } -}; - ShenandoahDirectCardMarkRememberedSet::ShenandoahDirectCardMarkRememberedSet(ShenandoahCardTable* card_table, size_t total_card_count) : LogCardValsPerIntPtr(log2i_exact(sizeof(intptr_t)) - log2i_exact(sizeof(CardValue))), LogCardSizeInWords(log2i_exact(CardTable::card_size_in_words())) { @@ -1039,38 +1078,44 @@ void ShenandoahReconstructRememberedSetTask::work(uint worker_id) { ShenandoahDirtyRememberedSetClosure dirty_cards_for_cross_generational_pointers; while (r != nullptr) { - if (r->is_old() && r->is_active()) { - HeapWord* obj_addr = r->bottom(); - if (r->is_humongous_start()) { - // First, clear the remembered set - oop obj = cast_to_oop(obj_addr); - size_t size = obj->size(); - - size_t num_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize); - size_t region_index = r->index(); - ShenandoahHeapRegion* humongous_region = heap->get_region(region_index); - while (num_regions-- != 0) { - scanner->reset_object_range(humongous_region->bottom(), humongous_region->end()); - region_index++; - humongous_region = heap->get_region(region_index); - } - - // Then register the humongous object and DIRTY relevant remembered set cards - scanner->register_object_without_lock(obj_addr); - obj->oop_iterate(&dirty_cards_for_cross_generational_pointers); - } else if (!r->is_humongous()) { - scanner->reset_object_range(r->bottom(), r->end()); - - // Then iterate over all objects, registering object and DIRTYing relevant remembered set cards - HeapWord* t = r->top(); - while (obj_addr < t) { + if (r->is_active()) { + if (r->is_old()) { + HeapWord* obj_addr = r->bottom(); + if (r->is_humongous_start()) { + // First, clear the remembered set oop obj = cast_to_oop(obj_addr); + size_t size = obj->size(); + + size_t num_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize); + size_t region_index = r->index(); + ShenandoahHeapRegion* humongous_region = heap->get_region(region_index); + while (num_regions-- != 0) { + scanner->reset_object_range(humongous_region->bottom(), humongous_region->end()); + region_index++; + humongous_region = heap->get_region(region_index); + } + + // Then register the humongous object and DIRTY relevant remembered set cards scanner->register_object_without_lock(obj_addr); - obj_addr += obj->oop_iterate_size(&dirty_cards_for_cross_generational_pointers); - } - } // else, ignore humongous continuation region + obj->oop_iterate(&dirty_cards_for_cross_generational_pointers); + } else if (!r->is_humongous()) { + scanner->reset_object_range(r->bottom(), r->end()); + + // Then iterate over all objects, registering object and DIRTYing relevant remembered set cards + HeapWord* t = r->top(); + while (obj_addr < t) { + oop obj = cast_to_oop(obj_addr); + scanner->register_object_without_lock(obj_addr); + obj_addr += obj->oop_iterate_size(&dirty_cards_for_cross_generational_pointers); + } + } // else, ignore humongous continuation region + } else { + // The region is young, but it may become old again and we don't want stale remembered set data. + assert(r->is_young(), "Region: %zu, is active but free", r->index()); + heap->old_generation()->clear_cards_for(r); + } } - // else, this region is FREE or YOUNG or inactive and we can ignore it. + // else, this region is FREE or inactive and we can ignore it. r = _regions->next(); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index c2c117e86e6..53f00e64a03 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -603,6 +603,9 @@ public: // as address. void register_object_without_lock(HeapWord* address); + // Dirty cards and register objects for the given range in memory. + void update_card_table(HeapWord* start, HeapWord* end); + // During the reference updates phase of GC, we walk through each old-gen memory region that was // not part of the collection set and we invalidate all unmarked objects. As part of this effort, // we coalesce neighboring dead objects in order to make future remembered set scanning more @@ -814,6 +817,10 @@ public: } } + void update_card_table(HeapWord* start, HeapWord* end) const { + _scc->update_card_table(start, end); + } + // Return true iff this object is "properly" registered. bool verify_registration(HeapWord* address, ShenandoahMarkingContext* ctx);