diff --git a/src/hotspot/share/gc/g1/g1CardSet.cpp b/src/hotspot/share/gc/g1/g1CardSet.cpp index 3441e6bc608..60ad63e812c 100644 --- a/src/hotspot/share/gc/g1/g1CardSet.cpp +++ b/src/hotspot/share/gc/g1/g1CardSet.cpp @@ -29,7 +29,6 @@ #include "gc/shared/gcLogPrecious.hpp" #include "gc/shared/gcTraceTime.inline.hpp" #include "memory/allocation.inline.hpp" -#include "runtime/atomicAccess.hpp" #include "runtime/globals_extension.hpp" #include "runtime/java.hpp" #include "utilities/bitMap.inline.hpp" @@ -192,32 +191,32 @@ const char* G1CardSetConfiguration::mem_object_type_name_str(uint index) { void G1CardSetCoarsenStats::reset() { STATIC_ASSERT(ARRAY_SIZE(_coarsen_from) == ARRAY_SIZE(_coarsen_collision)); for (uint i = 0; i < ARRAY_SIZE(_coarsen_from); i++) { - _coarsen_from[i] = 0; - _coarsen_collision[i] = 0; + _coarsen_from[i].store_relaxed(0); + _coarsen_collision[i].store_relaxed(0); } } void G1CardSetCoarsenStats::set(G1CardSetCoarsenStats& other) { STATIC_ASSERT(ARRAY_SIZE(_coarsen_from) == ARRAY_SIZE(_coarsen_collision)); for (uint i = 0; i < ARRAY_SIZE(_coarsen_from); i++) { - _coarsen_from[i] = other._coarsen_from[i]; - _coarsen_collision[i] = other._coarsen_collision[i]; + _coarsen_from[i].store_relaxed(other._coarsen_from[i].load_relaxed()); + _coarsen_collision[i].store_relaxed(other._coarsen_collision[i].load_relaxed()); } } void G1CardSetCoarsenStats::subtract_from(G1CardSetCoarsenStats& other) { STATIC_ASSERT(ARRAY_SIZE(_coarsen_from) == ARRAY_SIZE(_coarsen_collision)); for (uint i = 0; i < ARRAY_SIZE(_coarsen_from); i++) { - _coarsen_from[i] = other._coarsen_from[i] - _coarsen_from[i]; - _coarsen_collision[i] = other._coarsen_collision[i] - _coarsen_collision[i]; + _coarsen_from[i].store_relaxed(other._coarsen_from[i].load_relaxed() - _coarsen_from[i].load_relaxed()); + _coarsen_collision[i].store_relaxed(other._coarsen_collision[i].load_relaxed() - _coarsen_collision[i].load_relaxed()); } } void G1CardSetCoarsenStats::record_coarsening(uint tag, bool collision) { assert(tag < ARRAY_SIZE(_coarsen_from), "tag %u out of bounds", tag); - AtomicAccess::inc(&_coarsen_from[tag], memory_order_relaxed); + _coarsen_from[tag].add_then_fetch(1u, memory_order_relaxed); if (collision) { - AtomicAccess::inc(&_coarsen_collision[tag], memory_order_relaxed); + _coarsen_collision[tag].add_then_fetch(1u, memory_order_relaxed); } } @@ -228,13 +227,13 @@ void G1CardSetCoarsenStats::print_on(outputStream* out) { "Inline->AoC %zu (%zu) " "AoC->BitMap %zu (%zu) " "BitMap->Full %zu (%zu) ", - _coarsen_from[0], _coarsen_collision[0], - _coarsen_from[1], _coarsen_collision[1], + _coarsen_from[0].load_relaxed(), _coarsen_collision[0].load_relaxed(), + _coarsen_from[1].load_relaxed(), _coarsen_collision[1].load_relaxed(), // There is no BitMap at the first level so we can't . - _coarsen_from[3], _coarsen_collision[3], - _coarsen_from[4], _coarsen_collision[4], - _coarsen_from[5], _coarsen_collision[5], - _coarsen_from[6], _coarsen_collision[6] + _coarsen_from[3].load_relaxed(), _coarsen_collision[3].load_relaxed(), + _coarsen_from[4].load_relaxed(), _coarsen_collision[4].load_relaxed(), + _coarsen_from[5].load_relaxed(), _coarsen_collision[5].load_relaxed(), + _coarsen_from[6].load_relaxed(), _coarsen_collision[6].load_relaxed() ); } @@ -248,7 +247,7 @@ class G1CardSetHashTable : public CHeapObj { // the per region cardsets. const static uint GroupBucketClaimSize = 4; // Did we insert at least one card in the table? - bool volatile _inserted_card; + Atomic _inserted_card; G1CardSetMemoryManager* _mm; CardSetHash _table; @@ -311,10 +310,10 @@ public: G1CardSetHashTableValue value(region_idx, G1CardSetInlinePtr()); bool inserted = _table.insert_get(Thread::current(), lookup, value, found, should_grow); - if (!_inserted_card && inserted) { + if (!_inserted_card.load_relaxed() && inserted) { // It does not matter to us who is setting the flag so a regular atomic store // is sufficient. - AtomicAccess::store(&_inserted_card, true); + _inserted_card.store_relaxed(true); } return found.value(); @@ -343,9 +342,9 @@ public: } void reset() { - if (AtomicAccess::load(&_inserted_card)) { + if (_inserted_card.load_relaxed()) { _table.unsafe_reset(InitialLogTableSize); - AtomicAccess::store(&_inserted_card, false); + _inserted_card.store_relaxed(false); } } @@ -455,14 +454,14 @@ void G1CardSet::free_mem_object(ContainerPtr container) { _mm->free(container_type_to_mem_object_type(type), value); } -G1CardSet::ContainerPtr G1CardSet::acquire_container(ContainerPtr volatile* container_addr) { +G1CardSet::ContainerPtr G1CardSet::acquire_container(Atomic* container_addr) { // Update reference counts under RCU critical section to avoid a // use-after-cleapup bug where we increment a reference count for // an object whose memory has already been cleaned up and reused. GlobalCounter::CriticalSection cs(Thread::current()); while (true) { // Get ContainerPtr and increment refcount atomically wrt to memory reuse. - ContainerPtr container = AtomicAccess::load_acquire(container_addr); + ContainerPtr container = container_addr->load_acquire(); uint cs_type = container_type(container); if (container == FullCardSet || cs_type == ContainerInlinePtr) { return container; @@ -503,15 +502,15 @@ class G1ReleaseCardsets : public StackObj { G1CardSet* _card_set; using ContainerPtr = G1CardSet::ContainerPtr; - void coarsen_to_full(ContainerPtr* container_addr) { + void coarsen_to_full(Atomic* container_addr) { while (true) { - ContainerPtr cur_container = AtomicAccess::load_acquire(container_addr); + ContainerPtr cur_container = container_addr->load_acquire(); uint cs_type = G1CardSet::container_type(cur_container); if (cur_container == G1CardSet::FullCardSet) { return; } - ContainerPtr old_value = AtomicAccess::cmpxchg(container_addr, cur_container, G1CardSet::FullCardSet); + ContainerPtr old_value = container_addr->compare_exchange(cur_container, G1CardSet::FullCardSet); if (old_value == cur_container) { _card_set->release_and_maybe_free_container(cur_container); @@ -523,7 +522,7 @@ class G1ReleaseCardsets : public StackObj { public: explicit G1ReleaseCardsets(G1CardSet* card_set) : _card_set(card_set) { } - void operator ()(ContainerPtr* container_addr) { + void operator ()(Atomic* container_addr) { coarsen_to_full(container_addr); } }; @@ -544,10 +543,10 @@ G1AddCardResult G1CardSet::add_to_howl(ContainerPtr parent_container, ContainerPtr container; uint bucket = _config->howl_bucket_index(card_in_region); - ContainerPtr volatile* bucket_entry = howl->container_addr(bucket); + Atomic* bucket_entry = howl->container_addr(bucket); while (true) { - if (AtomicAccess::load(&howl->_num_entries) >= _config->cards_in_howl_threshold()) { + if (howl->_num_entries.load_relaxed() >= _config->cards_in_howl_threshold()) { return Overflow; } @@ -571,7 +570,7 @@ G1AddCardResult G1CardSet::add_to_howl(ContainerPtr parent_container, } if (increment_total && add_result == Added) { - AtomicAccess::inc(&howl->_num_entries, memory_order_relaxed); + howl->_num_entries.add_then_fetch(1u, memory_order_relaxed); } if (to_transfer != nullptr) { @@ -588,7 +587,7 @@ G1AddCardResult G1CardSet::add_to_bitmap(ContainerPtr container, uint card_in_re return bitmap->add(card_offset, _config->cards_in_howl_bitmap_threshold(), _config->max_cards_in_howl_bitmap()); } -G1AddCardResult G1CardSet::add_to_inline_ptr(ContainerPtr volatile* container_addr, ContainerPtr container, uint card_in_region) { +G1AddCardResult G1CardSet::add_to_inline_ptr(Atomic* container_addr, ContainerPtr container, uint card_in_region) { G1CardSetInlinePtr value(container_addr, container); return value.add(card_in_region, _config->inline_ptr_bits_per_card(), _config->max_cards_in_inline_ptr()); } @@ -610,7 +609,7 @@ G1CardSet::ContainerPtr G1CardSet::create_coarsened_array_of_cards(uint card_in_ return new_container; } -bool G1CardSet::coarsen_container(ContainerPtr volatile* container_addr, +bool G1CardSet::coarsen_container(Atomic* container_addr, ContainerPtr cur_container, uint card_in_region, bool within_howl) { @@ -640,7 +639,7 @@ bool G1CardSet::coarsen_container(ContainerPtr volatile* container_addr, ShouldNotReachHere(); } - ContainerPtr old_value = AtomicAccess::cmpxchg(container_addr, cur_container, new_container); // Memory order? + ContainerPtr old_value = container_addr->compare_exchange(cur_container, new_container); // Memory order? if (old_value == cur_container) { // Success. Indicate that the cards from the current card set must be transferred // by this caller. @@ -687,7 +686,7 @@ void G1CardSet::transfer_cards(G1CardSetHashTableValue* table_entry, ContainerPt assert(container_type(source_container) == ContainerHowl, "must be"); // Need to correct for that the Full remembered set occupies more cards than the // AoCS before. - AtomicAccess::add(&_num_occupied, _config->max_cards_in_region() - table_entry->_num_occupied, memory_order_relaxed); + _num_occupied.add_then_fetch(_config->max_cards_in_region() - table_entry->_num_occupied.load_relaxed(), memory_order_relaxed); } } @@ -713,18 +712,18 @@ void G1CardSet::transfer_cards_in_howl(ContainerPtr parent_container, diff -= 1; G1CardSetHowl* howling_array = container_ptr(parent_container); - AtomicAccess::add(&howling_array->_num_entries, diff, memory_order_relaxed); + howling_array->_num_entries.add_then_fetch(diff, memory_order_relaxed); G1CardSetHashTableValue* table_entry = get_container(card_region); assert(table_entry != nullptr, "Table entry not found for transferred cards"); - AtomicAccess::add(&table_entry->_num_occupied, diff, memory_order_relaxed); + table_entry->_num_occupied.add_then_fetch(diff, memory_order_relaxed); - AtomicAccess::add(&_num_occupied, diff, memory_order_relaxed); + _num_occupied.add_then_fetch(diff, memory_order_relaxed); } } -G1AddCardResult G1CardSet::add_to_container(ContainerPtr volatile* container_addr, +G1AddCardResult G1CardSet::add_to_container(Atomic* container_addr, ContainerPtr container, uint card_region, uint card_in_region, @@ -827,8 +826,8 @@ G1AddCardResult G1CardSet::add_card(uint card_region, uint card_in_region, bool } if (increment_total && add_result == Added) { - AtomicAccess::inc(&table_entry->_num_occupied, memory_order_relaxed); - AtomicAccess::inc(&_num_occupied, memory_order_relaxed); + table_entry->_num_occupied.add_then_fetch(1u, memory_order_relaxed); + _num_occupied.add_then_fetch(1u, memory_order_relaxed); } if (should_grow_table) { _table->grow(); @@ -853,7 +852,7 @@ bool G1CardSet::contains_card(uint card_region, uint card_in_region) { return false; } - ContainerPtr container = table_entry->_container; + ContainerPtr container = table_entry->_container.load_relaxed(); if (container == FullCardSet) { // contains_card() is not a performance critical method so we do not hide that // case in the switch below. @@ -889,7 +888,7 @@ void G1CardSet::print_info(outputStream* st, uintptr_t card) { return; } - ContainerPtr container = table_entry->_container; + ContainerPtr container = table_entry->_container.load_relaxed(); if (container == FullCardSet) { st->print("FULL card set)"); return; @@ -940,7 +939,7 @@ void G1CardSet::iterate_cards_during_transfer(ContainerPtr const container, Card void G1CardSet::iterate_containers(ContainerPtrClosure* cl, bool at_safepoint) { auto do_value = [&] (G1CardSetHashTableValue* value) { - cl->do_containerptr(value->_region_idx, value->_num_occupied, value->_container); + cl->do_containerptr(value->_region_idx, value->_num_occupied.load_relaxed(), value->_container.load_relaxed()); return true; }; @@ -1001,11 +1000,11 @@ bool G1CardSet::occupancy_less_or_equal_to(size_t limit) const { } bool G1CardSet::is_empty() const { - return _num_occupied == 0; + return _num_occupied.load_relaxed() == 0; } size_t G1CardSet::occupied() const { - return _num_occupied; + return _num_occupied.load_relaxed(); } size_t G1CardSet::num_containers() { @@ -1051,7 +1050,7 @@ size_t G1CardSet::static_mem_size() { void G1CardSet::clear() { _table->reset(); - _num_occupied = 0; + _num_occupied.store_relaxed(0); _mm->flush(); } diff --git a/src/hotspot/share/gc/g1/g1CardSet.hpp b/src/hotspot/share/gc/g1/g1CardSet.hpp index 9cefc4b1c22..64ddf0ca6a4 100644 --- a/src/hotspot/share/gc/g1/g1CardSet.hpp +++ b/src/hotspot/share/gc/g1/g1CardSet.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,6 +27,7 @@ #include "memory/allocation.hpp" #include "memory/memRegion.hpp" +#include "runtime/atomic.hpp" #include "utilities/concurrentHashTable.hpp" class G1CardSetAllocOptions; @@ -154,8 +155,8 @@ public: private: // Indices are "from" indices. - size_t _coarsen_from[NumCoarsenCategories]; - size_t _coarsen_collision[NumCoarsenCategories]; + Atomic _coarsen_from[NumCoarsenCategories]; + Atomic _coarsen_collision[NumCoarsenCategories]; public: G1CardSetCoarsenStats() { reset(); } @@ -271,11 +272,11 @@ private: // Total number of cards in this card set. This is a best-effort value, i.e. there may // be (slightly) more cards in the card set than this value in reality. - size_t _num_occupied; + Atomic _num_occupied; ContainerPtr make_container_ptr(void* value, uintptr_t type); - ContainerPtr acquire_container(ContainerPtr volatile* container_addr); + ContainerPtr acquire_container(Atomic* container_addr); // Returns true if the card set container should be released bool release_container(ContainerPtr container); // Release card set and free if needed. @@ -288,7 +289,7 @@ private: // coarsen_container does not transfer cards from cur_container // to the new container. Transfer is achieved by transfer_cards. // Returns true if this was the thread that coarsened the container (and added the card). - bool coarsen_container(ContainerPtr volatile* container_addr, + bool coarsen_container(Atomic* container_addr, ContainerPtr cur_container, uint card_in_region, bool within_howl = false); @@ -300,9 +301,9 @@ private: void transfer_cards(G1CardSetHashTableValue* table_entry, ContainerPtr source_container, uint card_region); void transfer_cards_in_howl(ContainerPtr parent_container, ContainerPtr source_container, uint card_region); - G1AddCardResult add_to_container(ContainerPtr volatile* container_addr, ContainerPtr container, uint card_region, uint card, bool increment_total = true); + G1AddCardResult add_to_container(Atomic* container_addr, ContainerPtr container, uint card_region, uint card, bool increment_total = true); - G1AddCardResult add_to_inline_ptr(ContainerPtr volatile* container_addr, ContainerPtr container, uint card_in_region); + G1AddCardResult add_to_inline_ptr(Atomic* container_addr, ContainerPtr container, uint card_in_region); G1AddCardResult add_to_array(ContainerPtr container, uint card_in_region); G1AddCardResult add_to_bitmap(ContainerPtr container, uint card_in_region); G1AddCardResult add_to_howl(ContainerPtr parent_container, uint card_region, uint card_in_region, bool increment_total = true); @@ -366,7 +367,6 @@ public: size_t num_containers(); - static G1CardSetCoarsenStats coarsen_stats(); static void print_coarsen_stats(outputStream* out); // Returns size of the actual remembered set containers in bytes. @@ -412,8 +412,15 @@ public: using ContainerPtr = G1CardSet::ContainerPtr; const uint _region_idx; - uint volatile _num_occupied; - ContainerPtr volatile _container; + Atomic _num_occupied; + Atomic _container; + + // Copy constructor needed for use in ConcurrentHashTable. + G1CardSetHashTableValue(const G1CardSetHashTableValue& other) : + _region_idx(other._region_idx), + _num_occupied(other._num_occupied.load_relaxed()), + _container(other._container.load_relaxed()) + { } G1CardSetHashTableValue(uint region_idx, ContainerPtr container) : _region_idx(region_idx), _num_occupied(0), _container(container) { } }; diff --git a/src/hotspot/share/gc/g1/g1CardSetContainers.hpp b/src/hotspot/share/gc/g1/g1CardSetContainers.hpp index 72c7795be2e..78551479e06 100644 --- a/src/hotspot/share/gc/g1/g1CardSetContainers.hpp +++ b/src/hotspot/share/gc/g1/g1CardSetContainers.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2023, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,7 +27,7 @@ #include "gc/g1/g1CardSet.hpp" #include "memory/allocation.hpp" -#include "runtime/atomicAccess.hpp" +#include "runtime/atomic.hpp" #include "utilities/bitMap.hpp" #include "utilities/globalDefinitions.hpp" @@ -67,7 +67,7 @@ class G1CardSetInlinePtr : public StackObj { using ContainerPtr = G1CardSet::ContainerPtr; - ContainerPtr volatile * _value_addr; + Atomic* _value_addr; ContainerPtr _value; static const uint SizeFieldLen = 3; @@ -103,7 +103,7 @@ public: explicit G1CardSetInlinePtr(ContainerPtr value) : G1CardSetInlinePtr(nullptr, value) {} - G1CardSetInlinePtr(ContainerPtr volatile* value_addr, ContainerPtr value) : _value_addr(value_addr), _value(value) { + G1CardSetInlinePtr(Atomic* value_addr, ContainerPtr value) : _value_addr(value_addr), _value(value) { assert(G1CardSet::container_type(_value) == G1CardSet::ContainerInlinePtr, "Value " PTR_FORMAT " is not a valid G1CardSetInlinePtr.", p2i(_value)); } @@ -145,13 +145,13 @@ public: // All but inline pointers are of this kind. For those, card entries are stored // directly in the ContainerPtr of the ConcurrentHashTable node. class G1CardSetContainer { - uintptr_t _ref_count; + Atomic _ref_count; protected: ~G1CardSetContainer() = default; public: G1CardSetContainer() : _ref_count(3) { } - uintptr_t refcount() const { return AtomicAccess::load_acquire(&_ref_count); } + uintptr_t refcount() const { return _ref_count.load_acquire(); } bool try_increment_refcount(); @@ -172,7 +172,7 @@ public: using ContainerPtr = G1CardSet::ContainerPtr; private: EntryCountType _size; - EntryCountType volatile _num_entries; + Atomic _num_entries; // VLA implementation. EntryDataType _data[1]; @@ -180,10 +180,10 @@ private: static const EntryCountType EntryMask = LockBitMask - 1; class G1CardSetArrayLocker : public StackObj { - EntryCountType volatile* _num_entries_addr; + Atomic* _num_entries_addr; EntryCountType _local_num_entries; public: - G1CardSetArrayLocker(EntryCountType volatile* value); + G1CardSetArrayLocker(Atomic* value); EntryCountType num_entries() const { return _local_num_entries; } void inc_num_entries() { @@ -192,7 +192,7 @@ private: } ~G1CardSetArrayLocker() { - AtomicAccess::release_store(_num_entries_addr, _local_num_entries); + _num_entries_addr->release_store(_local_num_entries); } }; @@ -213,7 +213,7 @@ public: template void iterate(CardVisitor& found); - size_t num_entries() const { return _num_entries & EntryMask; } + size_t num_entries() const { return _num_entries.load_relaxed() & EntryMask; } static size_t header_size_in_bytes(); @@ -223,7 +223,7 @@ public: }; class G1CardSetBitMap : public G1CardSetContainer { - size_t _num_bits_set; + Atomic _num_bits_set; BitMap::bm_word_t _bits[1]; public: @@ -236,7 +236,7 @@ public: return bm.at(card_idx); } - uint num_bits_set() const { return (uint)_num_bits_set; } + uint num_bits_set() const { return (uint)_num_bits_set.load_relaxed(); } template void iterate(CardVisitor& found, size_t const size_in_bits, uint offset); @@ -255,10 +255,10 @@ class G1CardSetHowl : public G1CardSetContainer { public: typedef uint EntryCountType; using ContainerPtr = G1CardSet::ContainerPtr; - EntryCountType volatile _num_entries; + Atomic _num_entries; private: // VLA implementation. - ContainerPtr _buckets[1]; + Atomic _buckets[1]; // Do not add class member variables beyond this point. // Iterates over the given ContainerPtr with at index in this Howl card set, @@ -268,14 +268,14 @@ private: ContainerPtr at(EntryCountType index) const; - ContainerPtr const* buckets() const; + Atomic const* buckets() const; public: G1CardSetHowl(EntryCountType card_in_region, G1CardSetConfiguration* config); - ContainerPtr const* container_addr(EntryCountType index) const; + Atomic const* container_addr(EntryCountType index) const; - ContainerPtr* container_addr(EntryCountType index); + Atomic* container_addr(EntryCountType index); bool contains(uint card_idx, G1CardSetConfiguration* config); // Iterates over all ContainerPtrs in this Howl card set, applying a CardOrRangeVisitor diff --git a/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp b/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp index 1958309f517..3c6fb9d1a02 100644 --- a/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp +++ b/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -67,7 +67,7 @@ inline G1AddCardResult G1CardSetInlinePtr::add(uint card_idx, uint bits_per_card return Overflow; } ContainerPtr new_value = merge(_value, card_idx, num_cards, bits_per_card); - ContainerPtr old_value = AtomicAccess::cmpxchg(_value_addr, _value, new_value, memory_order_relaxed); + ContainerPtr old_value = _value_addr->compare_exchange(_value, new_value, memory_order_relaxed); if (_value == old_value) { return Added; } @@ -126,7 +126,7 @@ inline bool G1CardSetContainer::try_increment_refcount() { } uintptr_t new_value = old_value + 2; - uintptr_t ref_count = AtomicAccess::cmpxchg(&_ref_count, old_value, new_value); + uintptr_t ref_count = _ref_count.compare_exchange(old_value, new_value); if (ref_count == old_value) { return true; } @@ -137,7 +137,7 @@ inline bool G1CardSetContainer::try_increment_refcount() { inline uintptr_t G1CardSetContainer::decrement_refcount() { uintptr_t old_value = refcount(); assert((old_value & 0x1) != 0 && old_value >= 3, "precondition"); - return AtomicAccess::sub(&_ref_count, 2u); + return _ref_count.sub_then_fetch(2u); } inline G1CardSetArray::G1CardSetArray(uint card_in_region, EntryCountType num_cards) : @@ -149,14 +149,13 @@ inline G1CardSetArray::G1CardSetArray(uint card_in_region, EntryCountType num_ca *entry_addr(0) = checked_cast(card_in_region); } -inline G1CardSetArray::G1CardSetArrayLocker::G1CardSetArrayLocker(EntryCountType volatile* num_entries_addr) : +inline G1CardSetArray::G1CardSetArrayLocker::G1CardSetArrayLocker(Atomic* num_entries_addr) : _num_entries_addr(num_entries_addr) { SpinYield s; - EntryCountType num_entries = AtomicAccess::load(_num_entries_addr) & EntryMask; + EntryCountType num_entries = _num_entries_addr->load_relaxed() & EntryMask; while (true) { - EntryCountType old_value = AtomicAccess::cmpxchg(_num_entries_addr, - num_entries, - (EntryCountType)(num_entries | LockBitMask)); + EntryCountType old_value = _num_entries_addr->compare_exchange(num_entries, + (EntryCountType)(num_entries | LockBitMask)); if (old_value == num_entries) { // Succeeded locking the array. _local_num_entries = num_entries; @@ -174,7 +173,7 @@ inline G1CardSetArray::EntryDataType const* G1CardSetArray::base_addr() const { } inline G1CardSetArray::EntryDataType const* G1CardSetArray::entry_addr(EntryCountType index) const { - assert(index < _num_entries, "precondition"); + assert(index < _num_entries.load_relaxed(), "precondition"); return base_addr() + index; } @@ -189,7 +188,7 @@ inline G1CardSetArray::EntryDataType G1CardSetArray::at(EntryCountType index) co inline G1AddCardResult G1CardSetArray::add(uint card_idx) { assert(card_idx < (1u << (sizeof(EntryDataType) * BitsPerByte)), "Card index %u does not fit allowed card value range.", card_idx); - EntryCountType num_entries = AtomicAccess::load_acquire(&_num_entries) & EntryMask; + EntryCountType num_entries = _num_entries.load_acquire() & EntryMask; EntryCountType idx = 0; for (; idx < num_entries; idx++) { if (at(idx) == card_idx) { @@ -223,7 +222,7 @@ inline G1AddCardResult G1CardSetArray::add(uint card_idx) { } inline bool G1CardSetArray::contains(uint card_idx) { - EntryCountType num_entries = AtomicAccess::load_acquire(&_num_entries) & EntryMask; + EntryCountType num_entries = _num_entries.load_acquire() & EntryMask; for (EntryCountType idx = 0; idx < num_entries; idx++) { if (at(idx) == card_idx) { @@ -235,7 +234,7 @@ inline bool G1CardSetArray::contains(uint card_idx) { template void G1CardSetArray::iterate(CardVisitor& found) { - EntryCountType num_entries = AtomicAccess::load_acquire(&_num_entries) & EntryMask; + EntryCountType num_entries = _num_entries.load_acquire() & EntryMask; for (EntryCountType idx = 0; idx < num_entries; idx++) { found(at(idx)); } @@ -256,11 +255,11 @@ inline G1CardSetBitMap::G1CardSetBitMap(uint card_in_region, uint size_in_bits) inline G1AddCardResult G1CardSetBitMap::add(uint card_idx, size_t threshold, size_t size_in_bits) { BitMapView bm(_bits, size_in_bits); - if (_num_bits_set >= threshold) { + if (_num_bits_set.load_relaxed() >= threshold) { return bm.at(card_idx) ? Found : Overflow; } if (bm.par_set_bit(card_idx)) { - AtomicAccess::inc(&_num_bits_set, memory_order_relaxed); + _num_bits_set.add_then_fetch(1u, memory_order_relaxed); return Added; } return Found; @@ -276,22 +275,22 @@ inline size_t G1CardSetBitMap::header_size_in_bytes() { return offset_of(G1CardSetBitMap, _bits); } -inline G1CardSetHowl::ContainerPtr const* G1CardSetHowl::container_addr(EntryCountType index) const { - assert(index < _num_entries, "precondition"); +inline Atomic const* G1CardSetHowl::container_addr(EntryCountType index) const { + assert(index < _num_entries.load_relaxed(), "precondition"); return buckets() + index; } -inline G1CardSetHowl::ContainerPtr* G1CardSetHowl::container_addr(EntryCountType index) { - return const_cast(const_cast(this)->container_addr(index)); +inline Atomic* G1CardSetHowl::container_addr(EntryCountType index) { + return const_cast*>(const_cast(this)->container_addr(index)); } inline G1CardSetHowl::ContainerPtr G1CardSetHowl::at(EntryCountType index) const { - return *container_addr(index); + return (*container_addr(index)).load_relaxed(); } -inline G1CardSetHowl::ContainerPtr const* G1CardSetHowl::buckets() const { +inline Atomic const* G1CardSetHowl::buckets() const { const void* ptr = reinterpret_cast(this) + header_size_in_bytes(); - return reinterpret_cast(ptr); + return reinterpret_cast const*>(ptr); } inline G1CardSetHowl::G1CardSetHowl(EntryCountType card_in_region, G1CardSetConfiguration* config) : @@ -300,7 +299,7 @@ inline G1CardSetHowl::G1CardSetHowl(EntryCountType card_in_region, G1CardSetConf EntryCountType num_buckets = config->num_buckets_in_howl(); EntryCountType bucket = config->howl_bucket_index(card_in_region); for (uint i = 0; i < num_buckets; ++i) { - *container_addr(i) = G1CardSetInlinePtr(); + container_addr(i)->store_relaxed(G1CardSetInlinePtr()); if (i == bucket) { G1CardSetInlinePtr value(container_addr(i), at(i)); value.add(card_in_region, config->inline_ptr_bits_per_card(), config->max_cards_in_inline_ptr()); @@ -310,8 +309,8 @@ inline G1CardSetHowl::G1CardSetHowl(EntryCountType card_in_region, G1CardSetConf inline bool G1CardSetHowl::contains(uint card_idx, G1CardSetConfiguration* config) { EntryCountType bucket = config->howl_bucket_index(card_idx); - ContainerPtr* array_entry = container_addr(bucket); - ContainerPtr container = AtomicAccess::load_acquire(array_entry); + Atomic* array_entry = container_addr(bucket); + ContainerPtr container = array_entry->load_acquire(); switch (G1CardSet::container_type(container)) { case G1CardSet::ContainerArrayOfCards: { diff --git a/src/hotspot/share/gc/g1/g1CardSetMemory.cpp b/src/hotspot/share/gc/g1/g1CardSetMemory.cpp index d13a6fe2dca..60602ef942b 100644 --- a/src/hotspot/share/gc/g1/g1CardSetMemory.cpp +++ b/src/hotspot/share/gc/g1/g1CardSetMemory.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,7 +26,6 @@ #include "gc/g1/g1CardSetContainers.inline.hpp" #include "gc/g1/g1CardSetMemory.inline.hpp" #include "gc/g1/g1MonotonicArena.inline.hpp" -#include "runtime/atomicAccess.hpp" #include "utilities/ostream.hpp" G1CardSetAllocator::G1CardSetAllocator(const char* name, diff --git a/test/hotspot/gtest/gc/g1/test_g1CardSetContainers.cpp b/test/hotspot/gtest/gc/g1/test_g1CardSetContainers.cpp index d83d8ffeef6..91951b35405 100644 --- a/test/hotspot/gtest/gc/g1/test_g1CardSetContainers.cpp +++ b/test/hotspot/gtest/gc/g1/test_g1CardSetContainers.cpp @@ -25,6 +25,7 @@ #include "gc/g1/g1HeapRegionBounds.inline.hpp" #include "gc/shared/cardTable.hpp" #include "memory/allocation.inline.hpp" +#include "runtime/atomic.hpp" #include "utilities/globalDefinitions.hpp" #include "utilities/powerOfTwo.hpp" #include "unittest.hpp" @@ -82,48 +83,48 @@ void G1CardSetContainersTest::cardset_inlineptr_test(uint bits_per_card) { G1AddCardResult res; - G1CardSet::ContainerPtr value = G1CardSetInlinePtr(); + Atomic value{}; for (uint i = 0; i < CardsPerSet; i++) { { - G1CardSetInlinePtr cards(&value, value); + G1CardSetInlinePtr cards(&value, value.load_relaxed()); res = cards.add(i + 1, bits_per_card, CardsPerSet); ASSERT_TRUE(res == Added); } { - G1CardSetInlinePtr cards(&value, value); + G1CardSetInlinePtr cards(&value, value.load_relaxed()); ASSERT_TRUE(cards.contains(i + 1, bits_per_card)); } } for (uint i = 0; i < CardsPerSet; i++) { - G1CardSetInlinePtr cards(value); + G1CardSetInlinePtr cards(value.load_relaxed()); ASSERT_TRUE(cards.contains(i + 1, bits_per_card)); } // Try to add again, should all return that the card had been added. for (uint i = 0; i < CardsPerSet; i++) { - G1CardSetInlinePtr cards(&value, value); + G1CardSetInlinePtr cards(&value, value.load_relaxed()); res = cards.add(i + 1, bits_per_card, CardsPerSet); ASSERT_TRUE(res == Found); } // Should be no more space in set. { - G1CardSetInlinePtr cards(&value, value); + G1CardSetInlinePtr cards(&value, value.load_relaxed()); res = cards.add(CardsPerSet + 1, bits_per_card, CardsPerSet); ASSERT_TRUE(res == Overflow); } // Cards should still be in the set. for (uint i = 0; i < CardsPerSet; i++) { - G1CardSetInlinePtr cards(value); + G1CardSetInlinePtr cards(value.load_relaxed()); ASSERT_TRUE(cards.contains(i + 1, bits_per_card)); } // Boundary cards should not be in the set. { - G1CardSetInlinePtr cards(value); + G1CardSetInlinePtr cards(value.load_relaxed()); ASSERT_TRUE(!cards.contains(0, bits_per_card)); ASSERT_TRUE(!cards.contains(CardsPerSet + 1, bits_per_card)); } @@ -131,7 +132,7 @@ void G1CardSetContainersTest::cardset_inlineptr_test(uint bits_per_card) { // Verify iteration finds all cards too and only those. { G1FindCardsInRange found(1, CardsPerSet); - G1CardSetInlinePtr cards(value); + G1CardSetInlinePtr cards(value.load_relaxed()); cards.iterate(found, bits_per_card); found.verify_all_found(); }