From 45246bc72daf68b905d7342f478e53bfe2883796 Mon Sep 17 00:00:00 2001 From: Kim Barrett Date: Thu, 20 Nov 2025 23:06:55 +0000 Subject: [PATCH] 8371965: Convert TaskQueueSuper to use Atomic Reviewed-by: iwalulya, tschatzl --- src/hotspot/share/gc/shared/taskqueue.hpp | 105 +++++++++++++--------- 1 file changed, 62 insertions(+), 43 deletions(-) diff --git a/src/hotspot/share/gc/shared/taskqueue.hpp b/src/hotspot/share/gc/shared/taskqueue.hpp index 1c36e18894a..3a751852ab6 100644 --- a/src/hotspot/share/gc/shared/taskqueue.hpp +++ b/src/hotspot/share/gc/shared/taskqueue.hpp @@ -25,13 +25,16 @@ #ifndef SHARE_GC_SHARED_TASKQUEUE_HPP #define SHARE_GC_SHARED_TASKQUEUE_HPP +#include "cppstdlib/type_traits.hpp" #include "memory/allocation.hpp" #include "memory/padded.hpp" +#include "metaprogramming/primitiveConversions.hpp" #include "oops/oopsHierarchy.hpp" -#include "runtime/atomicAccess.hpp" +#include "runtime/atomic.hpp" #include "utilities/debug.hpp" #include "utilities/globalDefinitions.hpp" #include "utilities/ostream.hpp" +#include "utilities/powerOfTwo.hpp" #include "utilities/stack.hpp" #if TASKQUEUE_STATS @@ -100,76 +103,92 @@ void TaskQueueStats::reset() { } #endif // TASKQUEUE_STATS +// Helper for TaskQueueSuper, encoding {queue index, tag} pair in a form that +// supports atomic access to the pair. +class TaskQueueAge { + friend struct PrimitiveConversions::Translate; + +public: + // Internal type used for indexing the queue, and for the tag. + using idx_t = NOT_LP64(uint16_t) LP64_ONLY(uint32_t); + + explicit TaskQueueAge(size_t data = 0) : _data{data} {} + TaskQueueAge(idx_t top, idx_t tag) : _fields{top, tag} {} + + idx_t top() const { return _fields._top; } + idx_t tag() const { return _fields._tag; } + + bool operator==(const TaskQueueAge& other) const { return _data == other._data; } + +private: + struct Fields { + idx_t _top; + idx_t _tag; + }; + union { + size_t _data; // Provides access to _fields as a single integral value. + Fields _fields; + }; + // _data must be able to hold combined _fields. Must be equal to ensure + // there isn't any padding that could be uninitialized by 2-arg ctor. + static_assert(sizeof(_data) == sizeof(_fields)); +}; + +// Support for Atomic. +template<> +struct PrimitiveConversions::Translate : public std::true_type { + using Value = TaskQueueAge; + using Decayed = decltype(TaskQueueAge::_data); + + static Decayed decay(Value x) { return x._data; } + static Value recover(Decayed x) { return Value(x); } +}; + // TaskQueueSuper collects functionality common to all GenericTaskQueue instances. template class TaskQueueSuper: public CHeapObj { protected: - // Internal type for indexing the queue; also used for the tag. - typedef NOT_LP64(uint16_t) LP64_ONLY(uint32_t) idx_t; - STATIC_ASSERT(N == idx_t(N)); // Ensure N fits in an idx_t. + using Age = TaskQueueAge; + using idx_t = Age::idx_t; + static_assert(N == idx_t(N)); // Ensure N fits in an idx_t. // N must be a power of 2 for computing modulo via masking. // N must be >= 2 for the algorithm to work at all, though larger is better. - STATIC_ASSERT(N >= 2); - STATIC_ASSERT(is_power_of_2(N)); + static_assert(N >= 2); + static_assert(is_power_of_2(N)); static const uint MOD_N_MASK = N - 1; - class Age { - friend class TaskQueueSuper; - - public: - explicit Age(size_t data = 0) : _data(data) {} - Age(idx_t top, idx_t tag) { _fields._top = top; _fields._tag = tag; } - - idx_t top() const { return _fields._top; } - idx_t tag() const { return _fields._tag; } - - bool operator ==(const Age& other) const { return _data == other._data; } - - private: - struct fields { - idx_t _top; - idx_t _tag; - }; - union { - size_t _data; - fields _fields; - }; - STATIC_ASSERT(sizeof(size_t) >= sizeof(fields)); - }; - uint bottom_relaxed() const { - return AtomicAccess::load(&_bottom); + return _bottom.load_relaxed(); } uint bottom_acquire() const { - return AtomicAccess::load_acquire(&_bottom); + return _bottom.load_acquire(); } void set_bottom_relaxed(uint new_bottom) { - AtomicAccess::store(&_bottom, new_bottom); + _bottom.store_relaxed(new_bottom); } void release_set_bottom(uint new_bottom) { - AtomicAccess::release_store(&_bottom, new_bottom); + _bottom.release_store(new_bottom); } Age age_relaxed() const { - return Age(AtomicAccess::load(&_age._data)); + return _age.load_relaxed(); } void set_age_relaxed(Age new_age) { - AtomicAccess::store(&_age._data, new_age._data); + _age.store_relaxed(new_age); } Age cmpxchg_age(Age old_age, Age new_age) { - return Age(AtomicAccess::cmpxchg(&_age._data, old_age._data, new_age._data)); + return _age.compare_exchange(old_age, new_age); } idx_t age_top_relaxed() const { - // Atomically accessing a subfield of an "atomic" member. - return AtomicAccess::load(&_age._fields._top); + return _age.load_relaxed().top(); } // These both operate mod N. @@ -222,16 +241,16 @@ private: DEFINE_PAD_MINUS_SIZE(0, DEFAULT_PADDING_SIZE, 0); // Index of the first free element after the last one pushed (mod N). - volatile uint _bottom; - DEFINE_PAD_MINUS_SIZE(1, DEFAULT_PADDING_SIZE, sizeof(uint)); + Atomic _bottom; + DEFINE_PAD_MINUS_SIZE(1, DEFAULT_PADDING_SIZE, sizeof(_bottom)); // top() is the index of the oldest pushed element (mod N), and tag() // is the associated epoch, to distinguish different modifications of // the age. There is no available element if top() == _bottom or // (_bottom - top()) mod N == N-1; the latter indicates underflow // during concurrent pop_local/pop_global. - volatile Age _age; - DEFINE_PAD_MINUS_SIZE(2, DEFAULT_PADDING_SIZE, sizeof(Age)); + Atomic _age; + DEFINE_PAD_MINUS_SIZE(2, DEFAULT_PADDING_SIZE, sizeof(_age)); NONCOPYABLE(TaskQueueSuper);