From de57d4b2e0fe3add0ef09945b34ddd0b67bbfa2b Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 17 May 2024 00:44:36 +0000 Subject: [PATCH] 8332257: Shenandoah: Move evacuation methods to implementation file Reviewed-by: shade, kdnilsen --- .../share/gc/shenandoah/shenandoahHeap.cpp | 77 +++++++++++++++++++ .../share/gc/shenandoah/shenandoahHeap.hpp | 2 +- .../gc/shenandoah/shenandoahHeap.inline.hpp | 77 ------------------- 3 files changed, 78 insertions(+), 78 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 8150df10914..f270e16f53e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1119,6 +1119,83 @@ void ShenandoahHeap::evacuate_collection_set(bool concurrent) { workers()->run_task(&task); } +oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { + if (ShenandoahThreadLocalData::is_oom_during_evac(Thread::current())) { + // This thread went through the OOM during evac protocol and it is safe to return + // the forward pointer. It must not attempt to evacuate any more. + return ShenandoahBarrierSet::resolve_forwarded(p); + } + + assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope"); + + size_t size = p->size(); + + assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects"); + + bool alloc_from_gclab = true; + HeapWord* copy = nullptr; + +#ifdef ASSERT + if (ShenandoahOOMDuringEvacALot && + (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call + copy = nullptr; + } else { +#endif + if (UseTLAB) { + copy = allocate_from_gclab(thread, size); + } + if (copy == nullptr) { + ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size); + copy = allocate_memory(req); + alloc_from_gclab = false; + } +#ifdef ASSERT + } +#endif + + if (copy == nullptr) { + control_thread()->handle_alloc_failure_evac(size); + + _oom_evac_handler.handle_out_of_memory_during_evacuation(); + + return ShenandoahBarrierSet::resolve_forwarded(p); + } + + // Copy the object: + Copy::aligned_disjoint_words(cast_from_oop(p), copy, size); + + // Try to install the new forwarding pointer. + oop copy_val = cast_to_oop(copy); + ContinuationGCSupport::relativize_stack_chunk(copy_val); + + oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val); + if (result == copy_val) { + // Successfully evacuated. Our copy is now the public one! + shenandoah_assert_correct(nullptr, copy_val); + return copy_val; + } else { + // Failed to evacuate. We need to deal with the object that is left behind. Since this + // new allocation is certainly after TAMS, it will be considered live in the next cycle. + // But if it happens to contain references to evacuated regions, those references would + // not get updated for this stale copy during this cycle, and we will crash while scanning + // it the next cycle. + // + // For GCLAB allocations, it is enough to rollback the allocation ptr. Either the next + // object will overwrite this stale copy, or the filler object on LAB retirement will + // do this. For non-GCLAB allocations, we have no way to retract the allocation, and + // have to explicitly overwrite the copy with the filler object. With that overwrite, + // we have to keep the fwdptr initialized and pointing to our (stale) copy. + if (alloc_from_gclab) { + ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size); + } else { + fill_with_object(copy, size); + shenandoah_assert_correct(nullptr, copy_val); + } + shenandoah_assert_correct(nullptr, result); + return result; + } +} + void ShenandoahHeap::trash_cset_regions() { ShenandoahHeapLocker locker(lock()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index e30b1d12e77..f3625da218c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -639,7 +639,7 @@ public: // Evacuates object src. Returns the evacuated object, either evacuated // by this thread, or by some other thread. - inline oop evacuate_object(oop src, Thread* thread); + oop evacuate_object(oop src, Thread* thread); // Call before/after evacuation. inline void enter_evacuation(Thread* t); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 87c3ea50ba6..f2cc602d8cb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -287,83 +287,6 @@ inline HeapWord* ShenandoahHeap::allocate_from_gclab(Thread* thread, size_t size return allocate_from_gclab_slow(thread, size); } -inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { - if (ShenandoahThreadLocalData::is_oom_during_evac(Thread::current())) { - // This thread went through the OOM during evac protocol and it is safe to return - // the forward pointer. It must not attempt to evacuate any more. - return ShenandoahBarrierSet::resolve_forwarded(p); - } - - assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope"); - - size_t size = p->size(); - - assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects"); - - bool alloc_from_gclab = true; - HeapWord* copy = nullptr; - -#ifdef ASSERT - if (ShenandoahOOMDuringEvacALot && - (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call - copy = nullptr; - } else { -#endif - if (UseTLAB) { - copy = allocate_from_gclab(thread, size); - } - if (copy == nullptr) { - ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size); - copy = allocate_memory(req); - alloc_from_gclab = false; - } -#ifdef ASSERT - } -#endif - - if (copy == nullptr) { - control_thread()->handle_alloc_failure_evac(size); - - _oom_evac_handler.handle_out_of_memory_during_evacuation(); - - return ShenandoahBarrierSet::resolve_forwarded(p); - } - - // Copy the object: - Copy::aligned_disjoint_words(cast_from_oop(p), copy, size); - - // Try to install the new forwarding pointer. - oop copy_val = cast_to_oop(copy); - ContinuationGCSupport::relativize_stack_chunk(copy_val); - - oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val); - if (result == copy_val) { - // Successfully evacuated. Our copy is now the public one! - shenandoah_assert_correct(nullptr, copy_val); - return copy_val; - } else { - // Failed to evacuate. We need to deal with the object that is left behind. Since this - // new allocation is certainly after TAMS, it will be considered live in the next cycle. - // But if it happens to contain references to evacuated regions, those references would - // not get updated for this stale copy during this cycle, and we will crash while scanning - // it the next cycle. - // - // For GCLAB allocations, it is enough to rollback the allocation ptr. Either the next - // object will overwrite this stale copy, or the filler object on LAB retirement will - // do this. For non-GCLAB allocations, we have no way to retract the allocation, and - // have to explicitly overwrite the copy with the filler object. With that overwrite, - // we have to keep the fwdptr initialized and pointing to our (stale) copy. - if (alloc_from_gclab) { - ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size); - } else { - fill_with_object(copy, size); - shenandoah_assert_correct(nullptr, copy_val); - } - shenandoah_assert_correct(nullptr, result); - return result; - } -} - inline bool ShenandoahHeap::requires_marking(const void* entry) const { oop obj = cast_to_oop(entry); return !_marking_context->is_marked_strong(obj);