From 5a01d002bd73e9ecef212f60df31a05e89ea2d03 Mon Sep 17 00:00:00 2001 From: Xiaolong Peng Date: Wed, 3 Jun 2026 02:04:01 -0700 Subject: [PATCH] fix: Make old-gen promotion budget reservation lock-free --- .../share/gc/shenandoah/shenandoahHeap.cpp | 6 ++++-- .../gc/shenandoah/shenandoahOldGeneration.cpp | 17 ++++++++++++----- .../gc/shenandoah/shenandoahOldGeneration.hpp | 5 +++-- .../shenandoah/test_shenandoahOldGeneration.cpp | 6 +++--- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 9cfe9e72331..0d62c91b406 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1039,8 +1039,10 @@ HeapWord* ShenandoahHeap::allocate_memory_work(ShenandoahAllocRequest& req, bool old_generation()->configure_plab_for_current_thread(req); } else if (req.is_promotion()) { const size_t actual_size = req.actual_size() * HeapWordSize; - log_debug(gc, plab)("Expend shared promotion of %zu bytes", actual_size); - old_generation()->expend_promoted(actual_size); + // Atomic reserve so concurrent GC workers can't over-expend the promotion budget without the lock. + if (!old_generation()->try_expend_promoted(actual_size)) { + log_debug(gc, plab)("Shared promotion of %zu bytes exceeded promotion reserve", actual_size); + } } } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 73ad316a892..52121e4b2ed 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -193,9 +193,17 @@ void ShenandoahOldGeneration::maybe_log_promotion_failure_stats(bool concurrent) } } -size_t ShenandoahOldGeneration::expend_promoted(size_t increment) { - assert(get_promoted_expended() + increment <= get_promoted_reserve(), "Do not expend more promotion than budgeted"); - return _promoted_expended.add_then_fetch(increment); +bool ShenandoahOldGeneration::try_expend_promoted(size_t increment) { + const size_t reserve = get_promoted_reserve(); + size_t cur = _promoted_expended.load_relaxed(); + while (cur + increment <= reserve) { + size_t prev = _promoted_expended.compare_exchange(cur, cur + increment); + if (prev == cur) { + return true; + } + cur = prev; + } + return false; } size_t ShenandoahOldGeneration::unexpend_promoted(size_t decrement) { @@ -244,12 +252,11 @@ ShenandoahOldGeneration::configure_plab_for_current_thread(const ShenandoahAlloc // The actual size of the allocation may be larger than the requested bytes (due to alignment on card boundaries). // If this puts us over our promotion budget, we need to disable future PLAB promotions for this thread. - if (can_promote(actual_size)) { + if (try_expend_promoted(actual_size)) { // Assume the entirety of this PLAB will be used for promotion. This prevents promotion from overreach. // When we retire this plab, we'll unexpend what we don't really use. log_debug(gc, plab)("Thread can promote using PLAB of %zu bytes. Expended: %zu, available: %zu", actual_size, get_promoted_expended(), get_promoted_reserve()); - expend_promoted(actual_size); shenandoah_plab->enable_promotions(); shenandoah_plab->set_actual_size(actual_size); } else { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp index 7e26d800e1d..eedf14ce62f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp @@ -111,8 +111,9 @@ public: // This zeros out the expended promotion count after the promotion reserve is computed void reset_promoted_expended(); - // This is incremented when allocations are made to copy promotions into the old generation - size_t expend_promoted(size_t increment); + // Atomically reserve `increment` bytes of promotion budget. Returns true iff the full amount + // was reserved without exceeding the reserve. Lock-free: safe to call without the heap lock. + bool try_expend_promoted(size_t increment); // This is used to return unused memory from a retired promotion LAB size_t unexpend_promoted(size_t decrement); diff --git a/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldGeneration.cpp b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldGeneration.cpp index 4633d8588d3..483464fba17 100644 --- a/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldGeneration.cpp +++ b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldGeneration.cpp @@ -55,7 +55,7 @@ protected: old = new ShenandoahOldGeneration(8); old->set_promoted_reserve(512 * HeapWordSize); - old->expend_promoted(256 * HeapWordSize); + old->try_expend_promoted(256 * HeapWordSize); old->set_evacuation_reserve(512 * HeapWordSize); Thread* thread = Thread::current(); @@ -171,10 +171,10 @@ TEST_VM_F(ShenandoahOldGenerationTest, test_actual_size_exceeds_promotion_reserv EXPECT_FALSE(promotions_enabled()) << "New plab can only be used for evacuations"; } -TEST_VM_F(ShenandoahOldGenerationTest, test_expend_promoted_should_increase_expended) { +TEST_VM_F(ShenandoahOldGenerationTest, test_try_expend_promoted_should_increase_expended) { SKIP_IF_NOT_SHENANDOAH(); size_t expended_before = old->get_promoted_expended(); - old->expend_promoted(128); + EXPECT_TRUE(old->try_expend_promoted(128)) << "Should fit within reserve"; size_t expended_after = old->get_promoted_expended(); EXPECT_EQ(expended_before + 128, expended_after) << "Should expend promotion"; }