8387806: Shenandoah: Reduce allocation-path contention from ShenandoahAllocRate byte accounting

Reviewed-by: shade, wkemper, kdnilsen

src/hotspot/share/gc/shenandoah/shenandoahAllocRate.hpp

@@ -25,7 +25,7 @@
 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHALLOCRATE_HPP
 #define SHARE_GC_SHENANDOAH_SHENANDOAHALLOCRATE_HPP
 
-#include "gc/shenandoah/shenandoahPadding.hpp"
+#include "gc/shenandoah/shenandoahStripedCounter.hpp"
 #include "gc/shenandoah/shenandoahWeightedSeq.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/mutex.hpp"
@@ -111,12 +111,26 @@ class ShenandoahAllocRate {
   static constexpr size_t ALLOC_SAMPLE_MAX = G;
 
   PaddedMonitor _sample_lock;
-  shenandoah_padding(0);
-  Atomic<size_t> _allocated_bytes_since_last_sample;
-  shenandoah_padding(1);
-  Atomic<size_t> _minimum_sample_size; // bytes, read by mutator, updated by gc
+  ShenandoahStripedCounter _unsampled;
+  // Packed minimum_sample_size and log_per_stripe_threshold for one alloc-path load.
+  Atomic<uint64_t> _sample_params;
   jlong _last_sample_time;
 
+  static uint64_t encode_sample_params(const uint32_t minimum_sample_size, const uint32_t log_per_stripe_threshold) {
+    return (static_cast<uint64_t>(log_per_stripe_threshold) << 32) |
+           minimum_sample_size;
+  }
+
+  static size_t decode_min_sample_size(const uint64_t params) {
+    return static_cast<uint32_t>(params);
+  }
+
+  static uint32_t decode_log_per_stripe_threshold(const uint64_t params) {
+    return static_cast<uint32_t>(params >> 32);
+  }
+
+  void maybe_take_sample(size_t minimum_sample_size, size_t striped_unsampled);
+
   ShenandoahWeightedSeq _baseline;
   ShenandoahWeightedSeq _recent;
   ShenandoahWeightedSeq _momentary;
@@ -127,22 +141,19 @@ public:
                                const uint recent_window_size = ShenandoahRecentAllocRateSampleWindow,
                                const uint momentary_window_size = ShenandoahMomentaryAllocRateSampleWindow)
     : _sample_lock(Mutex::nosafepoint - 2, "ShenandoahAllocSample_lock", true)
-    , _allocated_bytes_since_last_sample(0)
-    , _minimum_sample_size(minimum_sample_size)
     , _last_sample_time(Clock::elapsed_counter())
     , _baseline(baseline_window_size)
     , _recent(recent_window_size)
     , _momentary(momentary_window_size)
   {
+    set_minimum_sample_size(minimum_sample_size);
   }
 
   // Update minimum sample size based on the given available bytes
   void update_minimum_sample_size(size_t available);
 
-  // Set minimum sample size in bytes
-  void set_minimum_sample_size(const size_t minimum_sample_size) {
-    _minimum_sample_size.store_relaxed(minimum_sample_size);
-  }
+  // Set minimum sample size and its per-stripe trigger shift.
+  void set_minimum_sample_size(size_t minimum_sample_size);
 
   // Indicate that this many bytes have been allocated (by the mutator).
   void allocated(size_t allocated_bytes);
@@ -173,6 +184,24 @@ public:
   }
 
 private:
+  // Log2 of the per-stripe trigger threshold.
+  uint32_t log_per_stripe_threshold_for(size_t minimum_sample_size) const;
+
+  // Fast, lock-free: did this add carry the calling thread's stripe across a per-stripe threshold
+  // multiple? The threshold is a power of two, so a crossing is a change in the bits above it.
+  static bool striped_threshold_exceeded(size_t striped_unsampled, size_t previous_striped_unsampled, uint32_t log_per_stripe_threshold) {
+    return (striped_unsampled >> log_per_stripe_threshold) > (previous_striped_unsampled >> log_per_stripe_threshold);
+  }
+
+  // Whether the unsampled bytes are still below the sampling floor. Must be called under the sample
+  // lock: drains only happen under the lock, so reading the live stripe value and sum() here filters
+  // out false positives from a concurrent drain that already reset the counter.
+  bool unsampled_below_floor(size_t minimum_sample_size, size_t striped_unsampled) const {
+    assert(_sample_lock.owned_by_self(), "Caller must hold lock");
+    return (_unsampled.num_stripes() > 1 && _unsampled.current_stripe_value() < striped_unsampled) ||
+           _unsampled.sum() < minimum_sample_size;
+  }
+
   // Record the sample under the sample lock
   void take_sample(jlong now, jlong elapsed, size_t unsampled);
 

src/hotspot/share/gc/shenandoah/shenandoahAllocRate.inline.hpp

@@ -27,8 +27,10 @@
 
 #include "gc/shenandoah/shenandoahAllocRate.hpp"
 
+#include "gc/shenandoah/shenandoahStripedCounter.inline.hpp"
 #include "gc/shenandoah/shenandoahUtils.hpp"
 #include "logging/log.hpp"
+#include "utilities/powerOfTwo.hpp"
 
 
 inline size_t ShenandoahAnticipatedConsumption::baseline_consumption() const {
@@ -56,40 +58,56 @@ void ShenandoahAllocRate<Clock>::update_minimum_sample_size(const size_t availab
 }
 
 template<typename Clock>
-void ShenandoahAllocRate<Clock>::allocated(const size_t allocated_bytes) {
-  size_t unsampled = _allocated_bytes_since_last_sample.add_then_fetch(allocated_bytes, memory_order_relaxed);
-  const size_t minimum_sample_size = _minimum_sample_size.load_relaxed();
-  if (unsampled < minimum_sample_size) {
-    // Not enough to sample yet
-    return;
-  }
+uint32_t ShenandoahAllocRate<Clock>::log_per_stripe_threshold_for(const size_t minimum_sample_size) const {
+  // Floor-log2 of the per-stripe share. Clamps to 0 for a 1-byte trigger.
+  const int log_threshold = log2i(minimum_sample_size) - (int) _unsampled.log_num_stripes();
+  return log_threshold > 0 ? (uint32_t) log_threshold : 0u;
+}
 
+template<typename Clock>
+void ShenandoahAllocRate<Clock>::set_minimum_sample_size(const size_t minimum_sample_size) {
+  assert(minimum_sample_size > 0, "minimum sample size must be non-zero");
+  _sample_params.store_relaxed(encode_sample_params(checked_cast<uint32_t>(minimum_sample_size), log_per_stripe_threshold_for(minimum_sample_size)));
+}
+
+template<typename Clock>
+void ShenandoahAllocRate<Clock>::maybe_take_sample(const size_t minimum_sample_size, const size_t striped_unsampled) {
   if (!_sample_lock.try_lock()) {
-    // Another thread has the lock and will take the sample
+    // Another thread has the lock and will take the sample.
     return;
   }
 
-  unsampled = _allocated_bytes_since_last_sample.load_relaxed();
-  if (unsampled < minimum_sample_size) {
-    // Another thread has sampled and reset the allocated bytes under the lock
+  if (unsampled_below_floor(minimum_sample_size, striped_unsampled)) {
+    // Either another thread already sampled and drained, or this thread's stripe crossed its share
+    // while the aggregate is still short (skewed distribution). Wait for more.
     _sample_lock.unlock();
     return;
   }
-
   const jlong now = Clock::elapsed_counter();
   const jlong elapsed = now - _last_sample_time;
-
   if (elapsed <= 0) {
-    // Avoid sampling nonsense allocation rates
+    // Avoid sampling nonsense allocation rates.
     _sample_lock.unlock();
     return;
   }
-
-  take_sample(now, elapsed, unsampled);
-
+  take_sample(now, elapsed, _unsampled.drain());
   _sample_lock.unlock();
 }
 
+template<typename Clock>
+void ShenandoahAllocRate<Clock>::allocated(const size_t allocated_bytes) {
+  const size_t striped_unsampled = _unsampled.add(allocated_bytes);
+  const size_t previous_striped_unsampled = striped_unsampled - allocated_bytes;
+
+  const uint64_t params = _sample_params.load_relaxed();
+  const uint32_t log_per_stripe_threshold = decode_log_per_stripe_threshold(params);
+
+  // Re-arm the trigger at every per-stripe threshold crossing.
+  if (striped_threshold_exceeded(striped_unsampled, previous_striped_unsampled, log_per_stripe_threshold)) {
+    maybe_take_sample(decode_min_sample_size(params), striped_unsampled);
+  }
+}
+
 template<typename Clock>
 void ShenandoahAllocRate<Clock>::force_update() {
   if (!_sample_lock.try_lock()) {
@@ -97,7 +115,6 @@ void ShenandoahAllocRate<Clock>::force_update() {
     return;
   }
 
-  const size_t unsampled = _allocated_bytes_since_last_sample.load_relaxed();
   const jlong now = Clock::elapsed_counter();
   const jlong elapsed = now - _last_sample_time;
 
@@ -107,7 +124,7 @@ void ShenandoahAllocRate<Clock>::force_update() {
     return;
   }
 
-  take_sample(now, elapsed, unsampled);
+  take_sample(now, elapsed, _unsampled.drain());
 
   _sample_lock.unlock();
 }
@@ -118,10 +135,6 @@ void ShenandoahAllocRate<Clock>::take_sample(jlong now, jlong elapsed, size_t un
 
   _last_sample_time = now;
 
-  // We are recording this sample, deduct it from the counter. It may be increased
-  // concurrently by other threads outside the lock, so we still use an atomic access.
-  _allocated_bytes_since_last_sample.sub_then_fetch(unsampled, memory_order_relaxed);
-
   const double timestamp = static_cast<double>(_last_sample_time) / Clock::elapsed_frequency();
   const double rate_seconds = static_cast<double>(unsampled) * Clock::elapsed_frequency() / elapsed;
 

src/hotspot/share/gc/shenandoah/shenandoahStripedCounter.cpp

@@ -0,0 +1,38 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "gc/shenandoah/shenandoahStripedCounter.hpp"
+#include "memory/padded.inline.hpp"
+#include "runtime/os.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+ShenandoahStripedCounter::ShenandoahStripedCounter()
+  : _num_stripes(round_down_power_of_2((uint32_t) MAX2(os::processor_count(), 1)))
+  , _stripe_mask(_num_stripes - 1)
+  , _log_num_stripes(log2i_exact(_num_stripes)) {
+  _stripes = PaddedArray<Atomic<size_t>, mtGC>::create_unfreeable(_num_stripes);
+}
+
+ShenandoahStripedCounter::~ShenandoahStripedCounter() { }

src/hotspot/share/gc/shenandoah/shenandoahStripedCounter.hpp

@@ -0,0 +1,79 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHSTRIPEDCOUNTER_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHSTRIPEDCOUNTER_HPP
+
+#include "memory/allocation.hpp"
+#include "memory/padded.hpp"
+#include "runtime/atomic.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+// A contended-counter optimized for many concurrent writers and infrequent reads.
+// Each writer accumulates into a stripe chosen by its thread hash, each on its own cache line to
+// avoid false sharing. Stripes are shared when live writers outnumber stripes (num_stripes <= CPU
+// count). The value of the counter is always sum(stripes).
+//
+// Reads (sum) are approximate under concurrent writes and exact when quiescent.
+// This counter is monotonic per epoch: add() only increases it; drain() atomically reads and resets
+// to begin a new epoch (0), preserving concurrent adds that race with the drain.
+class ShenandoahStripedCounter : public CHeapObj<mtGC> {
+  typedef PaddedEnd<Atomic<size_t>> PaddedCounter;
+
+  PaddedCounter* _stripes;   // _num_stripes entries
+  // Number of stripes: a power of two, rounded down from the CPU count. Keeping it a power of two
+  // lets current_stripe() map a thread hash into range with a mask (& _stripe_mask) instead of a
+  // modulo on the hot path.
+  uint32_t const _num_stripes;
+  uint32_t const _stripe_mask; // _num_stripes - 1
+  uint32_t const _log_num_stripes;
+
+  // The stripe this thread uses.
+  uint32_t current_stripe() const;
+
+public:
+  ShenandoahStripedCounter();
+  ~ShenandoahStripedCounter();
+
+  // Add `bytes` to the current stripe of the counter and return the resulting total of the current stripe.
+  size_t add(size_t bytes);
+
+  // Current total of all stripes of the counter. No reset.
+  // Approximate under concurrent writes.
+  size_t sum() const;
+
+  // Current value of the calling thread's own stripe. O(1), no reset.
+  size_t current_stripe_value() const;
+
+  // Read the total and atomically reset it to zero, returning the amount consumed.
+  // Concurrent adds racing with the drain accumulate toward the next epoch rather than being lost.
+  size_t drain();
+
+  // Number of stripes (a power of two, <= CPU count), and its base-2 log. Exposed so a caller can
+  // scale a threshold to a per-stripe share with a shift (>> log_num_stripes) instead of a divide.
+  uint32_t num_stripes() const;
+  uint32_t log_num_stripes() const;
+};
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHSTRIPEDCOUNTER_HPP

src/hotspot/share/gc/shenandoah/shenandoahStripedCounter.inline.hpp

@@ -0,0 +1,74 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHSTRIPEDCOUNTER_INLINE_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHSTRIPEDCOUNTER_INLINE_HPP
+
+#include "gc/shenandoah/shenandoahStripedCounter.hpp"
+
+#include "runtime/thread.hpp"
+
+inline uint32_t ShenandoahStripedCounter::current_stripe() const {
+  if (_num_stripes == 1u) {
+    return 0u;
+  }
+  // Per-thread probe into [0, _num_stripes). Hashing the thread pointer spreads threads across
+  // stripes. This is a pure, stable function of (thread pointer,  _num_stripes)
+  const uintptr_t t = (uintptr_t) Thread::current();
+  return (uint32_t) ((t ^ (t >> 20) ^ (t >> 9)) & _stripe_mask);
+}
+
+inline uint32_t ShenandoahStripedCounter::num_stripes() const {
+  return _num_stripes;
+}
+
+inline uint32_t ShenandoahStripedCounter::log_num_stripes() const {
+  return _log_num_stripes;
+}
+
+inline size_t ShenandoahStripedCounter::add(const size_t bytes) {
+  return _stripes[current_stripe()].add_then_fetch(bytes, memory_order_relaxed);
+}
+
+inline size_t ShenandoahStripedCounter::sum() const {
+  size_t total = 0;
+  for (uint32_t i = 0; i < _num_stripes; i++) {
+    total += _stripes[i].load_relaxed();
+  }
+  return total;
+}
+
+inline size_t ShenandoahStripedCounter::current_stripe_value() const {
+  return _stripes[current_stripe()].load_relaxed();
+}
+
+inline size_t ShenandoahStripedCounter::drain() {
+  size_t total = 0;
+  for (uint32_t i = 0; i < _num_stripes; i++) {
+    total += _stripes[i].exchange(0, memory_order_relaxed);
+  }
+  return total;
+}
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHSTRIPEDCOUNTER_INLINE_HPP

test/hotspot/gtest/gc/shenandoah/test_shenandoahAllocationRate.cpp

@@ -26,6 +26,9 @@
 #include "gc/shared/gc_globals.hpp"
 
 #include "gc/shenandoah/shenandoahAllocRate.inline.hpp"
+#include "gc/shenandoah/shenandoahStripedCounter.inline.hpp"
+#include "runtime/atomic.hpp"
+#include "threadHelper.inline.hpp"
 
 class ShenandoahMockClock {
 public:
@@ -120,6 +123,131 @@ TEST_VM_F(ShenandoahAllocationRateTest, accelerated_consumption_momentary_spike)
   EXPECT_EQ(consumption.accelerated_consumption(), 0UL);
 }
 
+TEST_VM_F(ShenandoahAllocationRateTest, event_driven_sampling_single_dominant_allocator) {
+  // Single mutator: one stripe allocates, other stripes stay empty.
+  ShenandoahStripedCounter stripes;
+  if (stripes.num_stripes() == 1) {
+    // Regression requires multiple stripes.
+    return;
+  }
+
+  ShenandoahAllocRate<ShenandoahMockClock> rate(MINIMUM_SAMPLE_SIZE, BASELINE_SAMPLES, RECENT_SAMPLES, MOMENTARY_SAMPLES);
+  // Multiple epochs prove the allocation-path trigger re-fires without force_update().
+  constexpr size_t alloc_size = 64;
+  constexpr size_t epochs = 4;
+  for (size_t allocated = 0; allocated < MINIMUM_SAMPLE_SIZE * epochs; allocated += alloc_size) {
+    allocate(rate, alloc_size);
+  }
+
+  // Old one-shot trigger left the average at zero until force_update().
+  EXPECT_GT(rate.weighted_average(), 0.0);
+}
+
+TEST_VM_F(ShenandoahAllocationRateTest, event_driven_sampling_rearms_when_floor_lowered) {
+  // Lowering the floor must re-arm a stripe that crossed the old share.
+  constexpr size_t high_floor = 1 * M;
+  constexpr size_t low_floor = 1024;
+  constexpr size_t alloc_size = 64;
+
+  ShenandoahAllocRate<ShenandoahMockClock> rate(high_floor, BASELINE_SAMPLES, RECENT_SAMPLES, MOMENTARY_SAMPLES);
+
+  // Accumulate below the high floor, but above the later lowered share.
+  constexpr size_t phase1_bytes = high_floor / 4;
+  for (size_t allocated = 0; allocated < phase1_bytes; allocated += alloc_size) {
+    allocate(rate, alloc_size);
+  }
+  EXPECT_DOUBLE_EQ(rate.weighted_average(), 0.0);  // nothing drained yet
+
+  // A GC lowers the floor.
+  rate.set_minimum_sample_size(low_floor);
+
+  // New crossings under the lowered floor must sample without force_update().
+  for (size_t allocated = 0; allocated < low_floor * 16; allocated += alloc_size) {
+    allocate(rate, alloc_size);
+  }
+
+  EXPECT_GT(rate.weighted_average(), 0.0);
+}
+
+// Concurrent multi-threaded sampling. Many threads drive allocated() past the aggregate floor at
+// the same time, so distinct JavaThreads spread across stripes and stay hot simultaneously. This is
+// the regime the sampling guard is written for: contended try_lock (multiple threads cross their
+// per-stripe share at once, only one wins the lock), multi-stripe sum() aggregation (the floor is
+// reached by several occupied stripes, not one), and the drain-race clause (one thread's add()
+// captures a stripe value that another thread drains before the first takes the lock).
+class ConcurrentAllocators {
+public:
+  static constexpr int kThreads = 8;
+  static constexpr size_t kPerThreadEpochs = 500;
+  static constexpr size_t kAllocSize = 64;
+  // Every thread allocates this many bytes; the grand total spans many minimum-sample-size epochs.
+  static constexpr size_t kPerThreadBytes = MINIMUM_SAMPLE_SIZE * kPerThreadEpochs;
+};
+
+TEST_VM_F(ShenandoahAllocationRateTest, event_driven_sampling_concurrent_allocators) {
+  ShenandoahAllocRate<ShenandoahMockClock> rate(MINIMUM_SAMPLE_SIZE, BASELINE_SAMPLES, RECENT_SAMPLES, MOMENTARY_SAMPLES);
+
+  auto worker = [&](Thread*, int) {
+    for (size_t allocated = 0; allocated < ConcurrentAllocators::kPerThreadBytes;
+         allocated += ConcurrentAllocators::kAllocSize) {
+      rate.allocated(ConcurrentAllocators::kAllocSize);
+    }
+  };
+  TestThreadGroup<decltype(worker)> ttg(worker, ConcurrentAllocators::kThreads);
+  ttg.doit();
+  ttg.join();
+
+  // No force_update() was called: every sample came from the contended allocation path. Across
+  // thousands of epochs driven by all threads, sampling must have fired and drained repeatedly.
+  EXPECT_GT(rate.weighted_average(), 0.0);
+}
+
+// Concurrent skew: a few threads hold their stripes just below the per-stripe share and keep them
+// hot (spinning at the barrier), while a heavy thread pushes the aggregate over the floor. The
+// sample can then only be taken because sum() aggregates the heavy stripe with the held stripes --
+// exercising the multi-stripe floor crossing, not a single dominant stripe.
+class ConcurrentSkew {
+public:
+  static constexpr int kHolderThreads = 6;
+  static constexpr size_t kHeavyEpochs = 300;
+  static constexpr size_t kAllocSize = 64;
+};
+
+TEST_VM_F(ShenandoahAllocationRateTest, event_driven_sampling_concurrent_skew) {
+  ShenandoahStripedCounter stripes;
+  if (stripes.num_stripes() == 1) {
+    // A multi-stripe aggregate crossing is only meaningful with more than one stripe.
+    return;
+  }
+
+  ShenandoahAllocRate<ShenandoahMockClock> rate(MINIMUM_SAMPLE_SIZE, BASELINE_SAMPLES, RECENT_SAMPLES, MOMENTARY_SAMPLES);
+
+  // Each holder adds just under the per-stripe share once, then stays live for the whole run, so
+  // several stripes remain simultaneously occupied below their individual share. Their adds never
+  // cross a share alone, but they contend on the counter and feed sum().
+  Atomic<bool> stop(false);
+  const size_t per_stripe_share = MINIMUM_SAMPLE_SIZE / stripes.num_stripes();
+  const size_t holder_target = per_stripe_share > 2 ? per_stripe_share - 1 : 1;
+  auto holder = [&](Thread*, int) {
+    rate.allocated(holder_target);
+    while (!stop.load_relaxed()) { /* keep the thread (and its stripe) live */ }
+  };
+  TestThreadGroup<decltype(holder)> holders(holder, ConcurrentSkew::kHolderThreads);
+  holders.doit();
+
+  // Heavy stream on the main thread's own stripe. Its crossings, added to the held stripes, take
+  // sum() over the floor; the re-armed trigger must sample every epoch off the allocation path.
+  const size_t heavy_bytes = MINIMUM_SAMPLE_SIZE * ConcurrentSkew::kHeavyEpochs;
+  for (size_t allocated = 0; allocated < heavy_bytes; allocated += ConcurrentSkew::kAllocSize) {
+    allocate(rate, ConcurrentSkew::kAllocSize);
+  }
+
+  stop.store_relaxed(true);
+  holders.join();
+
+  EXPECT_GT(rate.weighted_average(), 0.0);
+}
+
 TEST_VM_F(ShenandoahAllocationRateTest, accelerated_consumption_accelerating) {
   ShenandoahAllocRate<ShenandoahMockClock> rate(256, BASELINE_SAMPLES, RECENT_SAMPLES, MOMENTARY_SAMPLES);
   for (uint i = 0; i < BASELINE_SAMPLES; ++i) {

test/hotspot/gtest/gc/shenandoah/test_shenandoahStripedCounter.cpp

@@ -0,0 +1,118 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "gc/shenandoah/shenandoahStripedCounter.inline.hpp"
+#include "runtime/atomic.hpp"
+#include "threadHelper.inline.hpp"
+#include "unittest.hpp"
+
+// Single thread: every add() maps to the same stripe, so add() returns the running total and
+// sum()/drain() are exact.
+TEST_VM(ShenandoahStripedCounter, single_thread_exact) {
+  ShenandoahStripedCounter c;
+  size_t expected = 0;
+  for (size_t i = 1; i <= 1000; i++) {
+    const size_t got = c.add(i);
+    expected += i;
+    // A lone writer owns one stripe, so its stripe total is the whole total.
+    EXPECT_EQ(got, expected);
+    EXPECT_EQ(c.sum(), expected);
+  }
+  // drain() returns everything and resets to zero; a second drain sees nothing.
+  EXPECT_EQ(c.drain(), expected);
+  EXPECT_EQ(c.sum(), (size_t) 0);
+  EXPECT_EQ(c.drain(), (size_t) 0);
+}
+
+// Draining mid-stream starts a fresh epoch, and sum()/drain() stay exact across the boundary.
+TEST_VM(ShenandoahStripedCounter, drain_epochs) {
+  ShenandoahStripedCounter c;
+  size_t expected = 0;
+  for (size_t i = 0; i < 500; i++) {
+    c.add(7);
+    expected += 7;
+  }
+  EXPECT_EQ(c.sum(), expected);
+  // Drain (starts a new epoch), then keep adding.
+  EXPECT_EQ(c.drain(), expected);
+  expected = 0;
+  for (size_t i = 0; i < 500; i++) {
+    c.add(13);
+    expected += 13;
+  }
+  EXPECT_EQ(c.sum(), expected);
+  EXPECT_EQ(c.drain(), expected);
+}
+
+// Multi-threaded stress. N threads each add a fixed number of bytes; when quiescent, sum() must
+// equal the grand total, and the periodic-drain variant must lose nothing (every byte lands in
+// exactly one drain or the final sum). Distinct JavaThreads make current_stripe() actually spread
+// writers across stripes.
+class StripedCounterStress {
+public:
+  static constexpr int kThreads = 8;
+  static constexpr size_t kPerThreadAdds = 20000;
+  static constexpr size_t kBytesPerAdd = 8;
+  static constexpr size_t kGrandTotal = (size_t) kThreads * kPerThreadAdds * kBytesPerAdd;
+};
+
+TEST_VM(ShenandoahStripedCounter, mt_quiescent_sum_exact) {
+  ShenandoahStripedCounter c;
+  auto worker = [&](Thread*, int) {
+    for (size_t i = 0; i < StripedCounterStress::kPerThreadAdds; i++) {
+      c.add(StripedCounterStress::kBytesPerAdd);
+    }
+  };
+  TestThreadGroup<decltype(worker)> ttg(worker, StripedCounterStress::kThreads);
+  ttg.doit();
+  ttg.join();
+  // All writers quiesced: sum() is now exact and must account for every byte.
+  EXPECT_EQ(c.sum(), StripedCounterStress::kGrandTotal);
+  EXPECT_EQ(c.drain(), StripedCounterStress::kGrandTotal);
+  EXPECT_EQ(c.sum(), (size_t) 0);
+}
+
+TEST_VM(ShenandoahStripedCounter, mt_concurrent_drain_loses_nothing) {
+  ShenandoahStripedCounter c;
+  Atomic<size_t> drained(0);
+  Atomic<int> done(0);
+  auto worker = [&](Thread*, int) {
+    for (size_t i = 0; i < StripedCounterStress::kPerThreadAdds; i++) {
+      c.add(StripedCounterStress::kBytesPerAdd);
+    }
+    done.add_then_fetch(1);
+  };
+  TestThreadGroup<decltype(worker)> ttg(worker, StripedCounterStress::kThreads);
+  ttg.doit();
+  // Drain concurrently with the adds; each drain moves bytes to a new epoch without losing them.
+  while (done.load_relaxed() < StripedCounterStress::kThreads) {
+    drained.add_then_fetch(c.drain());
+  }
+  ttg.join();
+  // Final drain sweeps up whatever raced the last concurrent drain.
+  drained.add_then_fetch(c.drain());
+  // Every byte added landed in exactly one drain.
+  EXPECT_EQ(drained.load_relaxed(), StripedCounterStress::kGrandTotal);
+  EXPECT_EQ(c.sum(), (size_t) 0);
+}