diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAllocRate.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAllocRate.hpp index 24221e504fd..ca94b91200a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAllocRate.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAllocRate.hpp @@ -25,7 +25,7 @@ #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHALLOCRATE_HPP #define SHARE_GC_SHENANDOAH_SHENANDOAHALLOCRATE_HPP -#include "gc/shenandoah/shenandoahPadding.hpp" +#include "gc/shenandoah/shenandoahStripedCounter.hpp" #include "gc/shenandoah/shenandoahWeightedSeq.hpp" #include "runtime/atomic.hpp" #include "runtime/mutex.hpp" @@ -111,12 +111,26 @@ class ShenandoahAllocRate { static constexpr size_t ALLOC_SAMPLE_MAX = G; PaddedMonitor _sample_lock; - shenandoah_padding(0); - Atomic _allocated_bytes_since_last_sample; - shenandoah_padding(1); - Atomic _minimum_sample_size; // bytes, read by mutator, updated by gc + ShenandoahStripedCounter _unsampled; + // Packed minimum_sample_size and log_per_stripe_threshold for one alloc-path load. + Atomic _sample_params; jlong _last_sample_time; + static uint64_t encode_sample_params(const uint32_t minimum_sample_size, const uint32_t log_per_stripe_threshold) { + return (static_cast(log_per_stripe_threshold) << 32) | + minimum_sample_size; + } + + static size_t decode_min_sample_size(const uint64_t params) { + return static_cast(params); + } + + static uint32_t decode_log_per_stripe_threshold(const uint64_t params) { + return static_cast(params >> 32); + } + + void maybe_take_sample(size_t minimum_sample_size, size_t striped_unsampled); + ShenandoahWeightedSeq _baseline; ShenandoahWeightedSeq _recent; ShenandoahWeightedSeq _momentary; @@ -127,22 +141,19 @@ public: const uint recent_window_size = ShenandoahRecentAllocRateSampleWindow, const uint momentary_window_size = ShenandoahMomentaryAllocRateSampleWindow) : _sample_lock(Mutex::nosafepoint - 2, "ShenandoahAllocSample_lock", true) - , _allocated_bytes_since_last_sample(0) - , _minimum_sample_size(minimum_sample_size) , _last_sample_time(Clock::elapsed_counter()) , _baseline(baseline_window_size) , _recent(recent_window_size) , _momentary(momentary_window_size) { + set_minimum_sample_size(minimum_sample_size); } // Update minimum sample size based on the given available bytes void update_minimum_sample_size(size_t available); - // Set minimum sample size in bytes - void set_minimum_sample_size(const size_t minimum_sample_size) { - _minimum_sample_size.store_relaxed(minimum_sample_size); - } + // Set minimum sample size and its per-stripe trigger shift. + void set_minimum_sample_size(size_t minimum_sample_size); // Indicate that this many bytes have been allocated (by the mutator). void allocated(size_t allocated_bytes); @@ -173,6 +184,24 @@ public: } private: + // Log2 of the per-stripe trigger threshold. + uint32_t log_per_stripe_threshold_for(size_t minimum_sample_size) const; + + // Fast, lock-free: did this add carry the calling thread's stripe across a per-stripe threshold + // multiple? The threshold is a power of two, so a crossing is a change in the bits above it. + static bool striped_threshold_exceeded(size_t striped_unsampled, size_t previous_striped_unsampled, uint32_t log_per_stripe_threshold) { + return (striped_unsampled >> log_per_stripe_threshold) > (previous_striped_unsampled >> log_per_stripe_threshold); + } + + // Whether the unsampled bytes are still below the sampling floor. Must be called under the sample + // lock: drains only happen under the lock, so reading the live stripe value and sum() here filters + // out false positives from a concurrent drain that already reset the counter. + bool unsampled_below_floor(size_t minimum_sample_size, size_t striped_unsampled) const { + assert(_sample_lock.owned_by_self(), "Caller must hold lock"); + return (_unsampled.num_stripes() > 1 && _unsampled.current_stripe_value() < striped_unsampled) || + _unsampled.sum() < minimum_sample_size; + } + // Record the sample under the sample lock void take_sample(jlong now, jlong elapsed, size_t unsampled); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAllocRate.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAllocRate.inline.hpp index 9ffad0d312c..e317721cd7b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAllocRate.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAllocRate.inline.hpp @@ -27,8 +27,10 @@ #include "gc/shenandoah/shenandoahAllocRate.hpp" +#include "gc/shenandoah/shenandoahStripedCounter.inline.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "logging/log.hpp" +#include "utilities/powerOfTwo.hpp" inline size_t ShenandoahAnticipatedConsumption::baseline_consumption() const { @@ -56,40 +58,56 @@ void ShenandoahAllocRate::update_minimum_sample_size(const size_t availab } template -void ShenandoahAllocRate::allocated(const size_t allocated_bytes) { - size_t unsampled = _allocated_bytes_since_last_sample.add_then_fetch(allocated_bytes, memory_order_relaxed); - const size_t minimum_sample_size = _minimum_sample_size.load_relaxed(); - if (unsampled < minimum_sample_size) { - // Not enough to sample yet - return; - } +uint32_t ShenandoahAllocRate::log_per_stripe_threshold_for(const size_t minimum_sample_size) const { + // Floor-log2 of the per-stripe share. Clamps to 0 for a 1-byte trigger. + const int log_threshold = log2i(minimum_sample_size) - (int) _unsampled.log_num_stripes(); + return log_threshold > 0 ? (uint32_t) log_threshold : 0u; +} +template +void ShenandoahAllocRate::set_minimum_sample_size(const size_t minimum_sample_size) { + assert(minimum_sample_size > 0, "minimum sample size must be non-zero"); + _sample_params.store_relaxed(encode_sample_params(checked_cast(minimum_sample_size), log_per_stripe_threshold_for(minimum_sample_size))); +} + +template +void ShenandoahAllocRate::maybe_take_sample(const size_t minimum_sample_size, const size_t striped_unsampled) { if (!_sample_lock.try_lock()) { - // Another thread has the lock and will take the sample + // Another thread has the lock and will take the sample. return; } - unsampled = _allocated_bytes_since_last_sample.load_relaxed(); - if (unsampled < minimum_sample_size) { - // Another thread has sampled and reset the allocated bytes under the lock + if (unsampled_below_floor(minimum_sample_size, striped_unsampled)) { + // Either another thread already sampled and drained, or this thread's stripe crossed its share + // while the aggregate is still short (skewed distribution). Wait for more. _sample_lock.unlock(); return; } - const jlong now = Clock::elapsed_counter(); const jlong elapsed = now - _last_sample_time; - if (elapsed <= 0) { - // Avoid sampling nonsense allocation rates + // Avoid sampling nonsense allocation rates. _sample_lock.unlock(); return; } - - take_sample(now, elapsed, unsampled); - + take_sample(now, elapsed, _unsampled.drain()); _sample_lock.unlock(); } +template +void ShenandoahAllocRate::allocated(const size_t allocated_bytes) { + const size_t striped_unsampled = _unsampled.add(allocated_bytes); + const size_t previous_striped_unsampled = striped_unsampled - allocated_bytes; + + const uint64_t params = _sample_params.load_relaxed(); + const uint32_t log_per_stripe_threshold = decode_log_per_stripe_threshold(params); + + // Re-arm the trigger at every per-stripe threshold crossing. + if (striped_threshold_exceeded(striped_unsampled, previous_striped_unsampled, log_per_stripe_threshold)) { + maybe_take_sample(decode_min_sample_size(params), striped_unsampled); + } +} + template void ShenandoahAllocRate::force_update() { if (!_sample_lock.try_lock()) { @@ -97,7 +115,6 @@ void ShenandoahAllocRate::force_update() { return; } - const size_t unsampled = _allocated_bytes_since_last_sample.load_relaxed(); const jlong now = Clock::elapsed_counter(); const jlong elapsed = now - _last_sample_time; @@ -107,7 +124,7 @@ void ShenandoahAllocRate::force_update() { return; } - take_sample(now, elapsed, unsampled); + take_sample(now, elapsed, _unsampled.drain()); _sample_lock.unlock(); } @@ -118,10 +135,6 @@ void ShenandoahAllocRate::take_sample(jlong now, jlong elapsed, size_t un _last_sample_time = now; - // We are recording this sample, deduct it from the counter. It may be increased - // concurrently by other threads outside the lock, so we still use an atomic access. - _allocated_bytes_since_last_sample.sub_then_fetch(unsampled, memory_order_relaxed); - const double timestamp = static_cast(_last_sample_time) / Clock::elapsed_frequency(); const double rate_seconds = static_cast(unsampled) * Clock::elapsed_frequency() / elapsed; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahStripedCounter.cpp b/src/hotspot/share/gc/shenandoah/shenandoahStripedCounter.cpp new file mode 100644 index 00000000000..d6e55d06248 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahStripedCounter.cpp @@ -0,0 +1,38 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "gc/shenandoah/shenandoahStripedCounter.hpp" +#include "memory/padded.inline.hpp" +#include "runtime/os.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/powerOfTwo.hpp" + +ShenandoahStripedCounter::ShenandoahStripedCounter() + : _num_stripes(round_down_power_of_2((uint32_t) MAX2(os::processor_count(), 1))) + , _stripe_mask(_num_stripes - 1) + , _log_num_stripes(log2i_exact(_num_stripes)) { + _stripes = PaddedArray, mtGC>::create_unfreeable(_num_stripes); +} + +ShenandoahStripedCounter::~ShenandoahStripedCounter() { } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahStripedCounter.hpp b/src/hotspot/share/gc/shenandoah/shenandoahStripedCounter.hpp new file mode 100644 index 00000000000..ad1086005a6 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahStripedCounter.hpp @@ -0,0 +1,79 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHSTRIPEDCOUNTER_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHSTRIPEDCOUNTER_HPP + +#include "memory/allocation.hpp" +#include "memory/padded.hpp" +#include "runtime/atomic.hpp" +#include "utilities/globalDefinitions.hpp" + +// A contended-counter optimized for many concurrent writers and infrequent reads. +// Each writer accumulates into a stripe chosen by its thread hash, each on its own cache line to +// avoid false sharing. Stripes are shared when live writers outnumber stripes (num_stripes <= CPU +// count). The value of the counter is always sum(stripes). +// +// Reads (sum) are approximate under concurrent writes and exact when quiescent. +// This counter is monotonic per epoch: add() only increases it; drain() atomically reads and resets +// to begin a new epoch (0), preserving concurrent adds that race with the drain. +class ShenandoahStripedCounter : public CHeapObj { + typedef PaddedEnd> PaddedCounter; + + PaddedCounter* _stripes; // _num_stripes entries + // Number of stripes: a power of two, rounded down from the CPU count. Keeping it a power of two + // lets current_stripe() map a thread hash into range with a mask (& _stripe_mask) instead of a + // modulo on the hot path. + uint32_t const _num_stripes; + uint32_t const _stripe_mask; // _num_stripes - 1 + uint32_t const _log_num_stripes; + + // The stripe this thread uses. + uint32_t current_stripe() const; + +public: + ShenandoahStripedCounter(); + ~ShenandoahStripedCounter(); + + // Add `bytes` to the current stripe of the counter and return the resulting total of the current stripe. + size_t add(size_t bytes); + + // Current total of all stripes of the counter. No reset. + // Approximate under concurrent writes. + size_t sum() const; + + // Current value of the calling thread's own stripe. O(1), no reset. + size_t current_stripe_value() const; + + // Read the total and atomically reset it to zero, returning the amount consumed. + // Concurrent adds racing with the drain accumulate toward the next epoch rather than being lost. + size_t drain(); + + // Number of stripes (a power of two, <= CPU count), and its base-2 log. Exposed so a caller can + // scale a threshold to a per-stripe share with a shift (>> log_num_stripes) instead of a divide. + uint32_t num_stripes() const; + uint32_t log_num_stripes() const; +}; + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHSTRIPEDCOUNTER_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahStripedCounter.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahStripedCounter.inline.hpp new file mode 100644 index 00000000000..58c40089324 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahStripedCounter.inline.hpp @@ -0,0 +1,74 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHSTRIPEDCOUNTER_INLINE_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHSTRIPEDCOUNTER_INLINE_HPP + +#include "gc/shenandoah/shenandoahStripedCounter.hpp" + +#include "runtime/thread.hpp" + +inline uint32_t ShenandoahStripedCounter::current_stripe() const { + if (_num_stripes == 1u) { + return 0u; + } + // Per-thread probe into [0, _num_stripes). Hashing the thread pointer spreads threads across + // stripes. This is a pure, stable function of (thread pointer, _num_stripes) + const uintptr_t t = (uintptr_t) Thread::current(); + return (uint32_t) ((t ^ (t >> 20) ^ (t >> 9)) & _stripe_mask); +} + +inline uint32_t ShenandoahStripedCounter::num_stripes() const { + return _num_stripes; +} + +inline uint32_t ShenandoahStripedCounter::log_num_stripes() const { + return _log_num_stripes; +} + +inline size_t ShenandoahStripedCounter::add(const size_t bytes) { + return _stripes[current_stripe()].add_then_fetch(bytes, memory_order_relaxed); +} + +inline size_t ShenandoahStripedCounter::sum() const { + size_t total = 0; + for (uint32_t i = 0; i < _num_stripes; i++) { + total += _stripes[i].load_relaxed(); + } + return total; +} + +inline size_t ShenandoahStripedCounter::current_stripe_value() const { + return _stripes[current_stripe()].load_relaxed(); +} + +inline size_t ShenandoahStripedCounter::drain() { + size_t total = 0; + for (uint32_t i = 0; i < _num_stripes; i++) { + total += _stripes[i].exchange(0, memory_order_relaxed); + } + return total; +} + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHSTRIPEDCOUNTER_INLINE_HPP diff --git a/test/hotspot/gtest/gc/shenandoah/test_shenandoahAllocationRate.cpp b/test/hotspot/gtest/gc/shenandoah/test_shenandoahAllocationRate.cpp index af066657377..a4169ff6ba6 100644 --- a/test/hotspot/gtest/gc/shenandoah/test_shenandoahAllocationRate.cpp +++ b/test/hotspot/gtest/gc/shenandoah/test_shenandoahAllocationRate.cpp @@ -26,6 +26,9 @@ #include "gc/shared/gc_globals.hpp" #include "gc/shenandoah/shenandoahAllocRate.inline.hpp" +#include "gc/shenandoah/shenandoahStripedCounter.inline.hpp" +#include "runtime/atomic.hpp" +#include "threadHelper.inline.hpp" class ShenandoahMockClock { public: @@ -120,6 +123,131 @@ TEST_VM_F(ShenandoahAllocationRateTest, accelerated_consumption_momentary_spike) EXPECT_EQ(consumption.accelerated_consumption(), 0UL); } +TEST_VM_F(ShenandoahAllocationRateTest, event_driven_sampling_single_dominant_allocator) { + // Single mutator: one stripe allocates, other stripes stay empty. + ShenandoahStripedCounter stripes; + if (stripes.num_stripes() == 1) { + // Regression requires multiple stripes. + return; + } + + ShenandoahAllocRate rate(MINIMUM_SAMPLE_SIZE, BASELINE_SAMPLES, RECENT_SAMPLES, MOMENTARY_SAMPLES); + // Multiple epochs prove the allocation-path trigger re-fires without force_update(). + constexpr size_t alloc_size = 64; + constexpr size_t epochs = 4; + for (size_t allocated = 0; allocated < MINIMUM_SAMPLE_SIZE * epochs; allocated += alloc_size) { + allocate(rate, alloc_size); + } + + // Old one-shot trigger left the average at zero until force_update(). + EXPECT_GT(rate.weighted_average(), 0.0); +} + +TEST_VM_F(ShenandoahAllocationRateTest, event_driven_sampling_rearms_when_floor_lowered) { + // Lowering the floor must re-arm a stripe that crossed the old share. + constexpr size_t high_floor = 1 * M; + constexpr size_t low_floor = 1024; + constexpr size_t alloc_size = 64; + + ShenandoahAllocRate rate(high_floor, BASELINE_SAMPLES, RECENT_SAMPLES, MOMENTARY_SAMPLES); + + // Accumulate below the high floor, but above the later lowered share. + constexpr size_t phase1_bytes = high_floor / 4; + for (size_t allocated = 0; allocated < phase1_bytes; allocated += alloc_size) { + allocate(rate, alloc_size); + } + EXPECT_DOUBLE_EQ(rate.weighted_average(), 0.0); // nothing drained yet + + // A GC lowers the floor. + rate.set_minimum_sample_size(low_floor); + + // New crossings under the lowered floor must sample without force_update(). + for (size_t allocated = 0; allocated < low_floor * 16; allocated += alloc_size) { + allocate(rate, alloc_size); + } + + EXPECT_GT(rate.weighted_average(), 0.0); +} + +// Concurrent multi-threaded sampling. Many threads drive allocated() past the aggregate floor at +// the same time, so distinct JavaThreads spread across stripes and stay hot simultaneously. This is +// the regime the sampling guard is written for: contended try_lock (multiple threads cross their +// per-stripe share at once, only one wins the lock), multi-stripe sum() aggregation (the floor is +// reached by several occupied stripes, not one), and the drain-race clause (one thread's add() +// captures a stripe value that another thread drains before the first takes the lock). +class ConcurrentAllocators { +public: + static constexpr int kThreads = 8; + static constexpr size_t kPerThreadEpochs = 500; + static constexpr size_t kAllocSize = 64; + // Every thread allocates this many bytes; the grand total spans many minimum-sample-size epochs. + static constexpr size_t kPerThreadBytes = MINIMUM_SAMPLE_SIZE * kPerThreadEpochs; +}; + +TEST_VM_F(ShenandoahAllocationRateTest, event_driven_sampling_concurrent_allocators) { + ShenandoahAllocRate rate(MINIMUM_SAMPLE_SIZE, BASELINE_SAMPLES, RECENT_SAMPLES, MOMENTARY_SAMPLES); + + auto worker = [&](Thread*, int) { + for (size_t allocated = 0; allocated < ConcurrentAllocators::kPerThreadBytes; + allocated += ConcurrentAllocators::kAllocSize) { + rate.allocated(ConcurrentAllocators::kAllocSize); + } + }; + TestThreadGroup ttg(worker, ConcurrentAllocators::kThreads); + ttg.doit(); + ttg.join(); + + // No force_update() was called: every sample came from the contended allocation path. Across + // thousands of epochs driven by all threads, sampling must have fired and drained repeatedly. + EXPECT_GT(rate.weighted_average(), 0.0); +} + +// Concurrent skew: a few threads hold their stripes just below the per-stripe share and keep them +// hot (spinning at the barrier), while a heavy thread pushes the aggregate over the floor. The +// sample can then only be taken because sum() aggregates the heavy stripe with the held stripes -- +// exercising the multi-stripe floor crossing, not a single dominant stripe. +class ConcurrentSkew { +public: + static constexpr int kHolderThreads = 6; + static constexpr size_t kHeavyEpochs = 300; + static constexpr size_t kAllocSize = 64; +}; + +TEST_VM_F(ShenandoahAllocationRateTest, event_driven_sampling_concurrent_skew) { + ShenandoahStripedCounter stripes; + if (stripes.num_stripes() == 1) { + // A multi-stripe aggregate crossing is only meaningful with more than one stripe. + return; + } + + ShenandoahAllocRate rate(MINIMUM_SAMPLE_SIZE, BASELINE_SAMPLES, RECENT_SAMPLES, MOMENTARY_SAMPLES); + + // Each holder adds just under the per-stripe share once, then stays live for the whole run, so + // several stripes remain simultaneously occupied below their individual share. Their adds never + // cross a share alone, but they contend on the counter and feed sum(). + Atomic stop(false); + const size_t per_stripe_share = MINIMUM_SAMPLE_SIZE / stripes.num_stripes(); + const size_t holder_target = per_stripe_share > 2 ? per_stripe_share - 1 : 1; + auto holder = [&](Thread*, int) { + rate.allocated(holder_target); + while (!stop.load_relaxed()) { /* keep the thread (and its stripe) live */ } + }; + TestThreadGroup holders(holder, ConcurrentSkew::kHolderThreads); + holders.doit(); + + // Heavy stream on the main thread's own stripe. Its crossings, added to the held stripes, take + // sum() over the floor; the re-armed trigger must sample every epoch off the allocation path. + const size_t heavy_bytes = MINIMUM_SAMPLE_SIZE * ConcurrentSkew::kHeavyEpochs; + for (size_t allocated = 0; allocated < heavy_bytes; allocated += ConcurrentSkew::kAllocSize) { + allocate(rate, ConcurrentSkew::kAllocSize); + } + + stop.store_relaxed(true); + holders.join(); + + EXPECT_GT(rate.weighted_average(), 0.0); +} + TEST_VM_F(ShenandoahAllocationRateTest, accelerated_consumption_accelerating) { ShenandoahAllocRate rate(256, BASELINE_SAMPLES, RECENT_SAMPLES, MOMENTARY_SAMPLES); for (uint i = 0; i < BASELINE_SAMPLES; ++i) { diff --git a/test/hotspot/gtest/gc/shenandoah/test_shenandoahStripedCounter.cpp b/test/hotspot/gtest/gc/shenandoah/test_shenandoahStripedCounter.cpp new file mode 100644 index 00000000000..db4933a9264 --- /dev/null +++ b/test/hotspot/gtest/gc/shenandoah/test_shenandoahStripedCounter.cpp @@ -0,0 +1,118 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "gc/shenandoah/shenandoahStripedCounter.inline.hpp" +#include "runtime/atomic.hpp" +#include "threadHelper.inline.hpp" +#include "unittest.hpp" + +// Single thread: every add() maps to the same stripe, so add() returns the running total and +// sum()/drain() are exact. +TEST_VM(ShenandoahStripedCounter, single_thread_exact) { + ShenandoahStripedCounter c; + size_t expected = 0; + for (size_t i = 1; i <= 1000; i++) { + const size_t got = c.add(i); + expected += i; + // A lone writer owns one stripe, so its stripe total is the whole total. + EXPECT_EQ(got, expected); + EXPECT_EQ(c.sum(), expected); + } + // drain() returns everything and resets to zero; a second drain sees nothing. + EXPECT_EQ(c.drain(), expected); + EXPECT_EQ(c.sum(), (size_t) 0); + EXPECT_EQ(c.drain(), (size_t) 0); +} + +// Draining mid-stream starts a fresh epoch, and sum()/drain() stay exact across the boundary. +TEST_VM(ShenandoahStripedCounter, drain_epochs) { + ShenandoahStripedCounter c; + size_t expected = 0; + for (size_t i = 0; i < 500; i++) { + c.add(7); + expected += 7; + } + EXPECT_EQ(c.sum(), expected); + // Drain (starts a new epoch), then keep adding. + EXPECT_EQ(c.drain(), expected); + expected = 0; + for (size_t i = 0; i < 500; i++) { + c.add(13); + expected += 13; + } + EXPECT_EQ(c.sum(), expected); + EXPECT_EQ(c.drain(), expected); +} + +// Multi-threaded stress. N threads each add a fixed number of bytes; when quiescent, sum() must +// equal the grand total, and the periodic-drain variant must lose nothing (every byte lands in +// exactly one drain or the final sum). Distinct JavaThreads make current_stripe() actually spread +// writers across stripes. +class StripedCounterStress { +public: + static constexpr int kThreads = 8; + static constexpr size_t kPerThreadAdds = 20000; + static constexpr size_t kBytesPerAdd = 8; + static constexpr size_t kGrandTotal = (size_t) kThreads * kPerThreadAdds * kBytesPerAdd; +}; + +TEST_VM(ShenandoahStripedCounter, mt_quiescent_sum_exact) { + ShenandoahStripedCounter c; + auto worker = [&](Thread*, int) { + for (size_t i = 0; i < StripedCounterStress::kPerThreadAdds; i++) { + c.add(StripedCounterStress::kBytesPerAdd); + } + }; + TestThreadGroup ttg(worker, StripedCounterStress::kThreads); + ttg.doit(); + ttg.join(); + // All writers quiesced: sum() is now exact and must account for every byte. + EXPECT_EQ(c.sum(), StripedCounterStress::kGrandTotal); + EXPECT_EQ(c.drain(), StripedCounterStress::kGrandTotal); + EXPECT_EQ(c.sum(), (size_t) 0); +} + +TEST_VM(ShenandoahStripedCounter, mt_concurrent_drain_loses_nothing) { + ShenandoahStripedCounter c; + Atomic drained(0); + Atomic done(0); + auto worker = [&](Thread*, int) { + for (size_t i = 0; i < StripedCounterStress::kPerThreadAdds; i++) { + c.add(StripedCounterStress::kBytesPerAdd); + } + done.add_then_fetch(1); + }; + TestThreadGroup ttg(worker, StripedCounterStress::kThreads); + ttg.doit(); + // Drain concurrently with the adds; each drain moves bytes to a new epoch without losing them. + while (done.load_relaxed() < StripedCounterStress::kThreads) { + drained.add_then_fetch(c.drain()); + } + ttg.join(); + // Final drain sweeps up whatever raced the last concurrent drain. + drained.add_then_fetch(c.drain()); + // Every byte added landed in exactly one drain. + EXPECT_EQ(drained.load_relaxed(), StripedCounterStress::kGrandTotal); + EXPECT_EQ(c.sum(), (size_t) 0); +}