diff --git a/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp b/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp index 4653f96980d..c695ad977fe 100644 --- a/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp +++ b/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -40,26 +40,26 @@ G1BlockOffsetTable::G1BlockOffsetTable(MemRegion heap, G1RegionToSpaceMapper* st MemRegion bot_reserved = storage->reserved(); - _offset_base = ((uint8_t*)bot_reserved.start() - (uintptr_t(_reserved.start()) >> CardTable::card_shift())); + _offset_base = ((Atomic*)bot_reserved.start() - (uintptr_t(_reserved.start()) >> CardTable::card_shift())); log_trace(gc, bot)("G1BlockOffsetTable::G1BlockOffsetTable: "); log_trace(gc, bot)(" rs.base(): " PTR_FORMAT " rs.size(): %zu rs end(): " PTR_FORMAT, p2i(bot_reserved.start()), bot_reserved.byte_size(), p2i(bot_reserved.end())); } -void G1BlockOffsetTable::set_offset_array(uint8_t* addr, uint8_t offset) { +void G1BlockOffsetTable::set_offset_array(Atomic* addr, uint8_t offset) { check_address(addr, "Block offset table address out of range"); - AtomicAccess::store(addr, offset); + addr->store_relaxed(offset); } -void G1BlockOffsetTable::set_offset_array(uint8_t* addr, HeapWord* high, HeapWord* low) { +void G1BlockOffsetTable::set_offset_array(Atomic* addr, HeapWord* high, HeapWord* low) { assert(high >= low, "addresses out of order"); size_t offset = pointer_delta(high, low); check_offset(offset, "offset too large"); set_offset_array(addr, (uint8_t)offset); } -void G1BlockOffsetTable::set_offset_array(uint8_t* left, uint8_t* right, uint8_t offset) { +void G1BlockOffsetTable::set_offset_array(Atomic* left, Atomic* right, uint8_t offset) { check_address(right, "Right block offset table address out of range"); assert(left <= right, "indexes out of order"); size_t num_cards = right - left + 1; @@ -67,9 +67,9 @@ void G1BlockOffsetTable::set_offset_array(uint8_t* left, uint8_t* right, uint8_t } #ifdef ASSERT -void G1BlockOffsetTable::check_address(uint8_t* addr, const char* msg) const { - uint8_t* start_addr = const_cast(_offset_base + (uintptr_t(_reserved.start()) >> CardTable::card_shift())); - uint8_t* end_addr = const_cast(_offset_base + (uintptr_t(_reserved.end()) >> CardTable::card_shift())); +void G1BlockOffsetTable::check_address(Atomic* addr, const char* msg) const { + Atomic* start_addr = const_cast*>(_offset_base + (uintptr_t(_reserved.start()) >> CardTable::card_shift())); + Atomic* end_addr = const_cast*>(_offset_base + (uintptr_t(_reserved.end()) >> CardTable::card_shift())); assert(addr >= start_addr && addr <= end_addr, "%s - offset address: " PTR_FORMAT ", start address: " PTR_FORMAT ", end address: " PTR_FORMAT, msg, (p2i(addr)), (p2i(start_addr)), (p2i(end_addr))); @@ -113,17 +113,17 @@ void G1BlockOffsetTable::check_address(uint8_t* addr, const char* msg) const { // Move back N (e.g., 8) entries and repeat with the // value of the new entry // -void G1BlockOffsetTable::set_remainder_to_point_to_start_incl(uint8_t* start_card, uint8_t* end_card) { +void G1BlockOffsetTable::set_remainder_to_point_to_start_incl(Atomic* start_card, Atomic* end_card) { assert(start_card <= end_card, "precondition"); assert(offset_array(start_card-1) < CardTable::card_size_in_words(), "Offset card has an unexpected value"); - uint8_t* start_card_for_region = start_card; + Atomic* start_card_for_region = start_card; uint8_t offset = UINT8_MAX; for (uint i = 0; i < BOTConstants::N_powers; i++) { // -1 so that the card with the actual offset is counted. Another -1 // so that the reach ends in this region and not at the start // of the next. - uint8_t* reach = start_card - 1 + (BOTConstants::power_to_cards_back(i+1) - 1); + Atomic* reach = start_card - 1 + (BOTConstants::power_to_cards_back(i+1) - 1); offset = CardTable::card_size_in_words() + i; if (reach >= end_card) { set_offset_array(start_card_for_region, end_card, offset); @@ -141,12 +141,12 @@ void G1BlockOffsetTable::set_remainder_to_point_to_start_incl(uint8_t* start_car // The card-interval [start_card, end_card] is a closed interval; this // is an expensive check -- use with care and only under protection of // suitable flag. -void G1BlockOffsetTable::check_all_cards(uint8_t* start_card, uint8_t* end_card) const { +void G1BlockOffsetTable::check_all_cards(Atomic* start_card, Atomic* end_card) const { if (end_card < start_card) { return; } guarantee(offset_array(start_card) == CardTable::card_size_in_words(), "Wrong value in second card"); - for (uint8_t* c = start_card + 1; c <= end_card; c++ /* yeah! */) { + for (Atomic* c = start_card + 1; c <= end_card; c++ /* yeah! */) { uint8_t entry = offset_array(c); if ((unsigned)(c - start_card) > BOTConstants::power_to_cards_back(1)) { guarantee(entry > CardTable::card_size_in_words(), @@ -157,7 +157,7 @@ void G1BlockOffsetTable::check_all_cards(uint8_t* start_card, uint8_t* end_card) (uint)entry, (uint)offset_array(c), CardTable::card_size_in_words()); } size_t backskip = BOTConstants::entry_to_cards_back(entry); - uint8_t* landing_card = c - backskip; + Atomic* landing_card = c - backskip; guarantee(landing_card >= (start_card - 1), "Inv"); if (landing_card >= start_card) { guarantee(offset_array(landing_card) <= entry, @@ -188,7 +188,7 @@ void G1BlockOffsetTable::check_all_cards(uint8_t* start_card, uint8_t* end_card) // void G1BlockOffsetTable::update_for_block_work(HeapWord* blk_start, HeapWord* blk_end) { HeapWord* const cur_card_boundary = align_up_by_card_size(blk_start); - uint8_t* const offset_card = entry_for_addr(cur_card_boundary); + Atomic* const offset_card = entry_for_addr(cur_card_boundary); assert(blk_start != nullptr && blk_end > blk_start, "phantom block"); @@ -209,7 +209,7 @@ void G1BlockOffsetTable::update_for_block_work(HeapWord* blk_start, HeapWord* bl // We need to now mark the subsequent cards that this block spans. // Index of card on which the block ends. - uint8_t* end_card = entry_for_addr(blk_end - 1); + Atomic* end_card = entry_for_addr(blk_end - 1); // Are there more cards left to be updated? if (offset_card + 1 <= end_card) { @@ -224,7 +224,7 @@ void G1BlockOffsetTable::update_for_block_work(HeapWord* blk_start, HeapWord* bl // The offset can be 0 if the block starts on a boundary. That // is checked by an assertion above. - uint8_t* previous_card = entry_for_addr(blk_start); + Atomic* previous_card = entry_for_addr(blk_start); HeapWord* boundary = addr_for_entry(previous_card); assert((offset_array(offset_card) == 0 && blk_start == boundary) || (offset_array(offset_card) > 0 && offset_array(offset_card) < CardTable::card_size_in_words()), @@ -240,7 +240,7 @@ void G1BlockOffsetTable::update_for_block_work(HeapWord* blk_start, HeapWord* bl } #ifdef ASSERT -void G1BlockOffsetTable::verify_offset(uint8_t* card_index, uint8_t upper_boundary) const { +void G1BlockOffsetTable::verify_offset(Atomic* card_index, uint8_t upper_boundary) const { assert(offset_array(card_index) <= upper_boundary, "Offset %u should not be larger than upper boundary %u.", (uint) offset_array(card_index), @@ -250,19 +250,19 @@ void G1BlockOffsetTable::verify_offset(uint8_t* card_index, uint8_t upper_bounda void G1BlockOffsetTable::verify_for_block(HeapWord* blk_start, HeapWord* blk_end) const { assert(is_crossing_card_boundary(blk_start, blk_end), "precondition"); - uint8_t* start_card = entry_for_addr(align_up_by_card_size(blk_start)); - uint8_t* end_card = entry_for_addr(blk_end - 1); + Atomic* start_card = entry_for_addr(align_up_by_card_size(blk_start)); + Atomic* end_card = entry_for_addr(blk_end - 1); // Check cards in [start_card, end_card] verify_offset(start_card, CardTable::card_size_in_words()); - for (uint8_t* current_card = start_card + 1; current_card <= end_card; ++current_card) { + for (Atomic* current_card = start_card + 1; current_card <= end_card; ++current_card) { assert(offset_array(current_card) > 0, "Offset %u is not larger than 0.", (uint) offset_array(current_card)); verify_offset(current_card, (uint8_t) (CardTable::card_size_in_words() + BOTConstants::N_powers - 1)); - uint8_t* prev = current_card - 1; - uint8_t* value = current_card; + Atomic* prev = current_card - 1; + Atomic* value = current_card; if (offset_array(prev) != offset_array(value)) { assert(offset_array(value) >= offset_array(prev), "monotonic"); size_t n_cards_back = BOTConstants::entry_to_cards_back(offset_array(value)); diff --git a/src/hotspot/share/gc/g1/g1BlockOffsetTable.hpp b/src/hotspot/share/gc/g1/g1BlockOffsetTable.hpp index 3b97efc4f0f..89c68ce96d2 100644 --- a/src/hotspot/share/gc/g1/g1BlockOffsetTable.hpp +++ b/src/hotspot/share/gc/g1/g1BlockOffsetTable.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,6 +29,7 @@ #include "gc/shared/blockOffsetTable.hpp" #include "gc/shared/cardTable.hpp" #include "memory/memRegion.hpp" +#include "runtime/atomic.hpp" #include "utilities/globalDefinitions.hpp" // This implementation of "G1BlockOffsetTable" divides the covered region @@ -41,7 +42,7 @@ private: MemRegion _reserved; // Biased array-start of BOT array for fast BOT entry translation - volatile uint8_t* _offset_base; + Atomic* _offset_base; void check_offset(size_t offset, const char* msg) const { assert(offset < CardTable::card_size_in_words(), @@ -51,32 +52,32 @@ private: // Bounds checking accessors: // For performance these have to devolve to array accesses in product builds. - inline uint8_t offset_array(uint8_t* addr) const; + inline uint8_t offset_array(Atomic* addr) const; - inline void set_offset_array(uint8_t* addr, uint8_t offset); + inline void set_offset_array(Atomic* addr, uint8_t offset); - inline void set_offset_array(uint8_t* addr, HeapWord* high, HeapWord* low); + inline void set_offset_array(Atomic* addr, HeapWord* high, HeapWord* low); - inline void set_offset_array(uint8_t* left, uint8_t* right, uint8_t offset); + inline void set_offset_array(Atomic* left, Atomic* right, uint8_t offset); // Mapping from address to object start array entry - inline uint8_t* entry_for_addr(const void* const p) const; + inline Atomic* entry_for_addr(const void* const p) const; // Mapping from object start array entry to address of first word - inline HeapWord* addr_for_entry(const uint8_t* const p) const; + inline HeapWord* addr_for_entry(const Atomic* const p) const; - void check_address(uint8_t* addr, const char* msg) const NOT_DEBUG_RETURN; + void check_address(Atomic* addr, const char* msg) const NOT_DEBUG_RETURN; // Sets the entries corresponding to the cards starting at "start" and ending // at "end" to point back to the card before "start"; [start, end] - void set_remainder_to_point_to_start_incl(uint8_t* start, uint8_t* end); + void set_remainder_to_point_to_start_incl(Atomic* start, Atomic* end); // Update BOT entries corresponding to the mem range [blk_start, blk_end). void update_for_block_work(HeapWord* blk_start, HeapWord* blk_end); - void check_all_cards(uint8_t* left_card, uint8_t* right_card) const NOT_DEBUG_RETURN; + void check_all_cards(Atomic* left_card, Atomic* right_card) const NOT_DEBUG_RETURN; - void verify_offset(uint8_t* card_index, uint8_t upper) const NOT_DEBUG_RETURN; + void verify_offset(Atomic* card_index, uint8_t upper) const NOT_DEBUG_RETURN; void verify_for_block(HeapWord* blk_start, HeapWord* blk_end) const NOT_DEBUG_RETURN; static HeapWord* align_up_by_card_size(HeapWord* const addr) { diff --git a/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp b/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp index 900e9516c1a..0d809b65526 100644 --- a/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp +++ b/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,12 +31,11 @@ #include "gc/shared/cardTable.hpp" #include "gc/shared/memset_with_concurrent_readers.hpp" #include "oops/oop.inline.hpp" -#include "runtime/atomicAccess.hpp" inline HeapWord* G1BlockOffsetTable::block_start_reaching_into_card(const void* addr) const { assert(_reserved.contains(addr), "invalid address"); - uint8_t* entry = entry_for_addr(addr); + Atomic* entry = entry_for_addr(addr); uint8_t offset = offset_array(entry); while (offset >= CardTable::card_size_in_words()) { // The excess of the offset from N_words indicates a power of Base @@ -50,19 +49,19 @@ inline HeapWord* G1BlockOffsetTable::block_start_reaching_into_card(const void* return q - offset; } -uint8_t G1BlockOffsetTable::offset_array(uint8_t* addr) const { +uint8_t G1BlockOffsetTable::offset_array(Atomic* addr) const { check_address(addr, "Block offset table address out of range"); - return AtomicAccess::load(addr); + return addr->load_relaxed(); } -inline uint8_t* G1BlockOffsetTable::entry_for_addr(const void* const p) const { +inline Atomic* G1BlockOffsetTable::entry_for_addr(const void* const p) const { assert(_reserved.contains(p), "out of bounds access to block offset table"); - uint8_t* result = const_cast(&_offset_base[uintptr_t(p) >> CardTable::card_shift()]); + Atomic* result = const_cast*>(&_offset_base[uintptr_t(p) >> CardTable::card_shift()]); return result; } -inline HeapWord* G1BlockOffsetTable::addr_for_entry(const uint8_t* const p) const { +inline HeapWord* G1BlockOffsetTable::addr_for_entry(const Atomic* const p) const { // _offset_base can be "negative", so can't use pointer_delta(). size_t delta = p - _offset_base; HeapWord* result = (HeapWord*) (delta << CardTable::card_shift());