From 6ae23a0d6574dc8139aea93ea3c562a7410fcb34 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 14 Jul 2026 23:41:25 +0000 Subject: [PATCH] 8388058: Shenandoah: Refactor arraycopy_work Reviewed-by: kdnilsen, shade --- .../gc/shenandoah/shenandoahBarrierSet.hpp | 10 +- .../shenandoahBarrierSet.inline.hpp | 113 +++++++++++------- 2 files changed, 76 insertions(+), 47 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp index 51b355e7042..ac896f24739 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp @@ -33,6 +33,7 @@ class ShenandoahHeap; class ShenandoahBarrierSetAssembler; class ShenandoahCardTable; +class ShenandoahMarkingContext; class ShenandoahBarrierSet: public BarrierSet { private: @@ -126,6 +127,10 @@ public: private: template void arraycopy_marking(T* dst, size_t count); + + template + bool is_above_tams(const ShenandoahMarkingContext* ctx, T* dst) const; + template inline void arraycopy_evacuation(T* src, size_t count); template @@ -134,10 +139,7 @@ private: template inline void clone_work(oop src); - template - inline void arraycopy_work(T* src, size_t count); - - inline bool need_bulk_update(HeapWord* dst); + inline bool need_bulk_update(HeapWord* dst) const; public: // Callbacks for runtime accesses. template diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp index f4b859afc44..af0622693fb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -509,40 +509,6 @@ OopCopyResult ShenandoahBarrierSet::AccessBarrier::oop_ return result; } -template -void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) { - // Young cycles are allowed to run when old marking is in progress. When old marking is in progress, - // this barrier will be called with ENQUEUE=true and HAS_FWD=false, even though the young generation - // may have forwarded objects. - assert(HAS_FWD == _heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(), "Forwarded object status is sane"); - // This function cannot be called to handle marking and evacuation at the same time (they operate on - // different sides of the copy). - static_assert((HAS_FWD || EVAC) != ENQUEUE, "Cannot evacuate and mark both sides of copy."); - - Thread* thread = Thread::current(); - SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread); - ShenandoahMarkingContext* ctx = _heap->marking_context(); - const ShenandoahCollectionSet* const cset = _heap->collection_set(); - T* end = src + count; - for (T* elem_ptr = src; elem_ptr < end; elem_ptr++) { - T o = RawAccess<>::oop_load(elem_ptr); - if (!CompressedOops::is_null(o)) { - oop obj = CompressedOops::decode_not_null(o); - if (HAS_FWD && cset->is_in(obj)) { - oop fwd = ShenandoahForwarding::get_forwardee(obj); - if (EVAC && obj == fwd) { - fwd = _heap->evacuate_object(obj, thread); - } - shenandoah_assert_forwarded_except(elem_ptr, obj, _heap->cancelled_gc()); - ShenandoahHeap::atomic_update_oop(fwd, elem_ptr, o); - } - if (ENQUEUE && !ctx->is_marked_strong(obj)) { - _satb_mark_queue_set.enqueue_known_active(queue, obj); - } - } - } -} - template void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) { if (count == 0) { @@ -574,31 +540,92 @@ void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) { template void ShenandoahBarrierSet::arraycopy_marking(T* dst, size_t count) { assert(_heap->is_concurrent_mark_in_progress(), "only during marking"); - if (ShenandoahSATBBarrier) { - if (!_heap->marking_context()->allocated_after_mark_start(reinterpret_cast(dst)) || - (IS_GENERATIONAL && _heap->heap_region_containing(dst)->is_old() && _heap->is_concurrent_young_mark_in_progress())) { - arraycopy_work(dst, count); + if (!ShenandoahSATBBarrier) { + return; + } + + const ShenandoahMarkingContext* ctx = _heap->marking_context(); + // Everything allocated above TAMS is alive and doesn't need the barrier to keep it that way + if (is_above_tams(ctx, dst)) { + return; + } + + assert(!_heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(), "Forwarded object status is sane"); + Thread* thread = Thread::current(); + SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread); + T* end = dst + count; + for (T* elem_ptr = dst; elem_ptr < end; ++elem_ptr) { + T o = RawAccess<>::oop_load(elem_ptr); + if (!CompressedOops::is_null(o)) { + oop obj = CompressedOops::decode_not_null(o); + if (!ctx->is_marked_strong(obj)) { + _satb_mark_queue_set.enqueue_known_active(queue, obj); + } } } } -inline bool ShenandoahBarrierSet::need_bulk_update(HeapWord* ary) { +template +bool ShenandoahBarrierSet::is_above_tams(const ShenandoahMarkingContext* ctx, T* dst) const { + // TAMS for an old region is unreliable during a young-only mark, so overwritten pointers in old dst arrays must + // be enqueued to preserve old->young referents copied in and overwritten after init mark. See JDK-8373116. + return ctx->allocated_after_mark_start(reinterpret_cast(dst)) + && !(IS_GENERATIONAL + && _heap->heap_region_containing(dst)->is_old() + && _heap->is_concurrent_young_mark_in_progress()); +} + +inline bool ShenandoahBarrierSet::need_bulk_update(HeapWord* ary) const { return ary < _heap->heap_region_containing(ary)->get_update_watermark(); } template void ShenandoahBarrierSet::arraycopy_evacuation(T* src, size_t count) { assert(_heap->is_evacuation_in_progress(), "only during evacuation"); - if (need_bulk_update(reinterpret_cast(src))) { - arraycopy_work(src, count); + if (!need_bulk_update(reinterpret_cast(src))) { + return; + } + + assert(_heap->has_forwarded_objects(), "Forwarded object status is sane"); + Thread* thread = Thread::current(); + const ShenandoahCollectionSet* const cset = _heap->collection_set(); + T* end = src + count; + for (T* elem_ptr = src; elem_ptr < end; ++elem_ptr) { + T o = RawAccess<>::oop_load(elem_ptr); + if (!CompressedOops::is_null(o)) { + oop obj = CompressedOops::decode_not_null(o); + if (cset->is_in(obj)) { + oop fwd = ShenandoahForwarding::get_forwardee(obj); + if (obj == fwd) { + fwd = _heap->evacuate_object(obj, thread); + } + shenandoah_assert_forwarded_except(elem_ptr, obj, _heap->cancelled_gc()); + ShenandoahHeap::atomic_update_oop(fwd, elem_ptr, o); + } + } } } template void ShenandoahBarrierSet::arraycopy_update(T* src, size_t count) { assert(_heap->is_update_refs_in_progress(), "only during update-refs"); - if (need_bulk_update(reinterpret_cast(src))) { - arraycopy_work(src, count); + if (!need_bulk_update(reinterpret_cast(src))) { + return; + } + + assert(_heap->has_forwarded_objects(), "Forwarded object status is sane"); + const ShenandoahCollectionSet* const cset = _heap->collection_set(); + T* end = src + count; + for (T* elem_ptr = src; elem_ptr < end; ++elem_ptr) { + T o = RawAccess<>::oop_load(elem_ptr); + if (!CompressedOops::is_null(o)) { + oop obj = CompressedOops::decode_not_null(o); + if (cset->is_in(obj)) { + oop fwd = ShenandoahForwarding::get_forwardee(obj); + shenandoah_assert_forwarded_except(elem_ptr, obj, _heap->cancelled_gc()); + ShenandoahHeap::atomic_update_oop(fwd, elem_ptr, o); + } + } } }