From 472fcb6e6e5fce4d5166a73894d1e7e4bd91bc58 Mon Sep 17 00:00:00 2001 From: Guoxiong Li Date: Tue, 26 Mar 2024 13:11:28 +0000 Subject: [PATCH] 8328508: Unify the signatures of the methods address_for_index() and index_for() in BOT implementations Reviewed-by: ayang, tschatzl --- .../share/gc/g1/g1BlockOffsetTable.cpp | 95 +++++++++---------- .../share/gc/g1/g1BlockOffsetTable.hpp | 35 +++---- .../share/gc/g1/g1BlockOffsetTable.inline.hpp | 71 ++++++-------- src/hotspot/share/gc/g1/g1RemSet.cpp | 2 +- .../gc/serial/serialBlockOffsetTable.cpp | 48 ++++------ .../gc/serial/serialBlockOffsetTable.hpp | 31 +++--- .../serial/serialBlockOffsetTable.inline.hpp | 22 ++--- .../share/gc/serial/vmStructs_serial.hpp | 2 +- src/hotspot/share/gc/shared/cardTable.cpp | 2 - src/hotspot/share/gc/shared/cardTable.hpp | 5 - 10 files changed, 133 insertions(+), 180 deletions(-) diff --git a/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp b/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp index b0334031263..d252bc176e1 100644 --- a/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp +++ b/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp @@ -36,11 +36,11 @@ ////////////////////////////////////////////////////////////////////// G1BlockOffsetTable::G1BlockOffsetTable(MemRegion heap, G1RegionToSpaceMapper* storage) : - _reserved(heap), _offset_array(nullptr) { + _reserved(heap), _offset_base(nullptr) { MemRegion bot_reserved = storage->reserved(); - _offset_array = (u_char*)bot_reserved.start(); + _offset_base = ((u_char*)bot_reserved.start() - (uintptr_t(_reserved.start()) >> CardTable::card_shift())); log_trace(gc, bot)("G1BlockOffsetTable::G1BlockOffsetTable: "); log_trace(gc, bot)(" rs.base(): " PTR_FORMAT " rs.size(): " SIZE_FORMAT " rs end(): " PTR_FORMAT, @@ -48,16 +48,17 @@ G1BlockOffsetTable::G1BlockOffsetTable(MemRegion heap, G1RegionToSpaceMapper* st } #ifdef ASSERT -void G1BlockOffsetTable::check_index(size_t index, const char* msg) const { - assert((index) < (_reserved.word_size() >> CardTable::card_shift_in_words()), - "%s - index: " SIZE_FORMAT ", _vs.committed_size: " SIZE_FORMAT, - msg, (index), (_reserved.word_size() >> CardTable::card_shift_in_words())); - assert(G1CollectedHeap::heap()->is_in(address_for_index_raw(index)), - "Index " SIZE_FORMAT " corresponding to " PTR_FORMAT +void G1BlockOffsetTable::check_address(u_char* addr, const char* msg) const { + u_char* start_addr = const_cast(_offset_base + (uintptr_t(_reserved.start()) >> CardTable::card_shift())); + u_char* end_addr = const_cast(_offset_base + (uintptr_t(_reserved.end()) >> CardTable::card_shift())); + assert(addr >= start_addr && addr <= end_addr, + "%s - offset address: " PTR_FORMAT ", start address: " PTR_FORMAT ", end address: " PTR_FORMAT, + msg, (p2i(addr)), (p2i(start_addr)), (p2i(end_addr))); + assert(G1CollectedHeap::heap()->is_in(addr_for_entry(addr)), + "Offset address " PTR_FORMAT " corresponding to " PTR_FORMAT " (%u) is not in committed area.", - (index), - p2i(address_for_index_raw(index)), - G1CollectedHeap::heap()->addr_to_region(address_for_index_raw(index))); + (p2i(addr)), p2i(addr_for_entry(addr)), + G1CollectedHeap::heap()->addr_to_region(addr_for_entry(addr))); } #endif // ASSERT @@ -67,9 +68,7 @@ void G1BlockOffsetTable::check_index(size_t index, const char* msg) const { G1BlockOffsetTablePart::G1BlockOffsetTablePart(G1BlockOffsetTable* array, HeapRegion* hr) : _bot(array), - _hr(hr) -{ -} + _hr(hr) {} // Write the backskip value for each region. // @@ -96,7 +95,6 @@ G1BlockOffsetTablePart::G1BlockOffsetTablePart(G1BlockOffsetTable* array, HeapRe // the index of the entry // // Given an address, -// Find the index for the address // Find the block offset table entry // Convert the entry to a back slide // (e.g., with today's, offset = 0x81 => @@ -104,18 +102,18 @@ G1BlockOffsetTablePart::G1BlockOffsetTablePart(G1BlockOffsetTable* array, HeapRe // Move back N (e.g., 8) entries and repeat with the // value of the new entry // -void G1BlockOffsetTablePart::set_remainder_to_point_to_start_incl(size_t start_card, size_t end_card) { +void G1BlockOffsetTablePart::set_remainder_to_point_to_start_incl(u_char* start_card, u_char* end_card) { assert(start_card <= end_card, "precondition"); - assert(start_card > _bot->index_for(_hr->bottom()), "Cannot be first card"); + assert(start_card > _bot->entry_for_addr(_hr->bottom()), "Cannot be first card"); assert(_bot->offset_array(start_card-1) < CardTable::card_size_in_words(), "Offset card has an unexpected value"); - size_t start_card_for_region = start_card; + u_char* start_card_for_region = start_card; u_char offset = max_jubyte; for (uint i = 0; i < BOTConstants::N_powers; i++) { // -1 so that the card with the actual offset is counted. Another -1 // so that the reach ends in this region and not at the start // of the next. - size_t reach = start_card - 1 + (BOTConstants::power_to_cards_back(i+1) - 1); + u_char* reach = start_card - 1 + (BOTConstants::power_to_cards_back(i+1) - 1); offset = CardTable::card_size_in_words() + i; if (reach >= end_card) { _bot->set_offset_array(start_card_for_region, end_card, offset); @@ -133,15 +131,14 @@ void G1BlockOffsetTablePart::set_remainder_to_point_to_start_incl(size_t start_c // The card-interval [start_card, end_card] is a closed interval; this // is an expensive check -- use with care and only under protection of // suitable flag. -void G1BlockOffsetTablePart::check_all_cards(size_t start_card, size_t end_card) const { - +void G1BlockOffsetTablePart::check_all_cards(u_char* start_card, u_char* end_card) const { if (end_card < start_card) { return; } guarantee(_bot->offset_array(start_card) == CardTable::card_size_in_words(), "Wrong value in second card"); - for (size_t c = start_card + 1; c <= end_card; c++ /* yeah! */) { + for (u_char* c = start_card + 1; c <= end_card; c++ /* yeah! */) { u_char entry = _bot->offset_array(c); - if (c - start_card > BOTConstants::power_to_cards_back(1)) { + if ((unsigned)(c - start_card) > BOTConstants::power_to_cards_back(1)) { guarantee(entry > CardTable::card_size_in_words(), "Should be in logarithmic region - " "entry: %u, " @@ -150,7 +147,7 @@ void G1BlockOffsetTablePart::check_all_cards(size_t start_card, size_t end_card) (uint)entry, (uint)_bot->offset_array(c), CardTable::card_size_in_words()); } size_t backskip = BOTConstants::entry_to_cards_back(entry); - size_t landing_card = c - backskip; + u_char* landing_card = c - backskip; guarantee(landing_card >= (start_card - 1), "Inv"); if (landing_card >= start_card) { guarantee(_bot->offset_array(landing_card) <= entry, @@ -182,7 +179,7 @@ void G1BlockOffsetTablePart::check_all_cards(size_t start_card, size_t end_card) void G1BlockOffsetTablePart::update_for_block_work(HeapWord* blk_start, HeapWord* blk_end) { HeapWord* const cur_card_boundary = align_up_by_card_size(blk_start); - size_t const index = _bot->index_for_raw(cur_card_boundary); + u_char* const offset_card = _bot->entry_for_addr(cur_card_boundary); assert(blk_start != nullptr && blk_end > blk_start, "phantom block"); @@ -194,41 +191,41 @@ void G1BlockOffsetTablePart::update_for_block_work(HeapWord* blk_start, "reference must be into the heap"); assert(G1CollectedHeap::heap()->is_in_reserved(blk_end - 1), "limit must be within the heap"); - assert(cur_card_boundary == _bot->_reserved.start() + index * CardTable::card_size_in_words(), - "index must agree with cur_card_boundary"); + assert(cur_card_boundary == _bot->addr_for_entry(offset_card), + "Block offset table entry must agree with cur_card_boundary"); // Mark the card that holds the offset into the block. - _bot->set_offset_array(index, cur_card_boundary, blk_start); + _bot->set_offset_array(offset_card, cur_card_boundary, blk_start); // We need to now mark the subsequent cards that this block spans. // Index of card on which the block ends. - size_t end_index = _bot->index_for(blk_end - 1); + u_char* end_card = _bot->entry_for_addr(blk_end - 1); // Are there more cards left to be updated? - if (index + 1 <= end_index) { - set_remainder_to_point_to_start_incl(index + 1, end_index); + if (offset_card + 1 <= end_card) { + set_remainder_to_point_to_start_incl(offset_card + 1, end_card); } #ifdef ASSERT // Calculate new_card_boundary this way because end_index // may be the last valid index in the covered region. - HeapWord* new_card_boundary = _bot->address_for_index(end_index) + CardTable::card_size_in_words(); + HeapWord* new_card_boundary = _bot->addr_for_entry(end_card) + CardTable::card_size_in_words(); assert(new_card_boundary >= blk_end, "postcondition"); // The offset can be 0 if the block starts on a boundary. That // is checked by an assertion above. - size_t start_index = _bot->index_for(blk_start); - HeapWord* boundary = _bot->address_for_index(start_index); - assert((_bot->offset_array(index) == 0 && blk_start == boundary) || - (_bot->offset_array(index) > 0 && _bot->offset_array(index) < CardTable::card_size_in_words()), + u_char* previous_card = _bot->entry_for_addr(blk_start); + HeapWord* boundary = _bot->addr_for_entry(previous_card); + assert((_bot->offset_array(offset_card) == 0 && blk_start == boundary) || + (_bot->offset_array(offset_card) > 0 && _bot->offset_array(offset_card) < CardTable::card_size_in_words()), "offset array should have been set - " "index offset: %u, " "blk_start: " PTR_FORMAT ", " "boundary: " PTR_FORMAT, - (uint)_bot->offset_array(index), + (uint)_bot->offset_array(offset_card), p2i(blk_start), p2i(boundary)); - for (size_t j = index + 1; j <= end_index; j++) { + for (u_char* j = offset_card + 1; j <= end_card; j++) { assert(_bot->offset_array(j) > 0 && _bot->offset_array(j) <= (u_char) (CardTable::card_size_in_words() + BOTConstants::N_powers - 1), @@ -243,16 +240,16 @@ void G1BlockOffsetTablePart::update_for_block_work(HeapWord* blk_start, void G1BlockOffsetTablePart::verify() const { assert(_hr->bottom() < _hr->top(), "Only non-empty regions should be verified."); - size_t start_card = _bot->index_for(_hr->bottom()); - size_t end_card = _bot->index_for(_hr->top() - 1); + u_char* start_card = _bot->entry_for_addr(_hr->bottom()); + u_char* end_card = _bot->entry_for_addr(_hr->top() - 1); - for (size_t current_card = start_card; current_card < end_card; current_card++) { + for (u_char* current_card = start_card; current_card < end_card; current_card++) { u_char entry = _bot->offset_array(current_card); if (entry < CardTable::card_size_in_words()) { // The entry should point to an object before the current card. Verify that // it is possible to walk from that object in to the current card by just // iterating over the objects following it. - HeapWord* card_address = _bot->address_for_index(current_card); + HeapWord* card_address = _bot->addr_for_entry(current_card); HeapWord* obj_end = card_address - entry; while (obj_end < card_address) { HeapWord* obj = obj_end; @@ -272,9 +269,9 @@ void G1BlockOffsetTablePart::verify() const { size_t max_backskip = current_card - start_card; guarantee(backskip <= max_backskip, "Going backwards beyond the start_card. start_card: " SIZE_FORMAT " current_card: " SIZE_FORMAT " backskip: " SIZE_FORMAT, - start_card, current_card, backskip); + p2i(start_card), p2i(current_card), backskip); - HeapWord* backskip_address = _bot->address_for_index(current_card - backskip); + HeapWord* backskip_address = _bot->addr_for_entry(current_card - backskip); guarantee(backskip_address >= _hr->bottom(), "Going backwards beyond bottom of the region: bottom: " PTR_FORMAT ", backskip_address: " PTR_FORMAT, p2i(_hr->bottom()), p2i(backskip_address)); @@ -284,14 +281,14 @@ void G1BlockOffsetTablePart::verify() const { #ifndef PRODUCT void G1BlockOffsetTablePart::print_on(outputStream* out) { - size_t from_index = _bot->index_for(_hr->bottom()); - size_t to_index = _bot->index_for(_hr->end()); + u_char* from_card = _bot->entry_for_addr(_hr->bottom()); + u_char* to_card = _bot->entry_for_addr(_hr->end()); out->print_cr(">> BOT for area [" PTR_FORMAT "," PTR_FORMAT ") " "cards [" SIZE_FORMAT "," SIZE_FORMAT ")", - p2i(_hr->bottom()), p2i(_hr->end()), from_index, to_index); - for (size_t i = from_index; i < to_index; ++i) { + p2i(_hr->bottom()), p2i(_hr->end()), p2i(from_card), p2i(to_card)); + for (u_char* i = from_card; i < to_card; ++i) { out->print_cr(" entry " SIZE_FORMAT_W(8) " | " PTR_FORMAT " : %3u", - i, p2i(_bot->address_for_index(i)), + p2i(i), p2i(_bot->addr_for_entry(i)), (uint) _bot->offset_array(i)); } } diff --git a/src/hotspot/share/gc/g1/g1BlockOffsetTable.hpp b/src/hotspot/share/gc/g1/g1BlockOffsetTable.hpp index 5858702f8db..98f3232b6c5 100644 --- a/src/hotspot/share/gc/g1/g1BlockOffsetTable.hpp +++ b/src/hotspot/share/gc/g1/g1BlockOffsetTable.hpp @@ -51,9 +51,8 @@ private: // The reserved region covered by the table. MemRegion _reserved; - // Array for keeping offsets for retrieving object start fast given an - // address. - volatile u_char* _offset_array; // byte array keeping backwards offsets + // Biased array-start of BOT array for fast BOT entry translation + volatile u_char* _offset_base; void check_offset(size_t offset, const char* msg) const { assert(offset < CardTable::card_size_in_words(), @@ -63,16 +62,16 @@ private: // Bounds checking accessors: // For performance these have to devolve to array accesses in product builds. - inline u_char offset_array(size_t index) const; + inline u_char offset_array(u_char* addr) const; - inline void set_offset_array_raw(size_t index, u_char offset); - inline void set_offset_array(size_t index, u_char offset); + inline void set_offset_array_raw(u_char* addr, u_char offset); + inline void set_offset_array(u_char* addr, u_char offset); - inline void set_offset_array(size_t index, HeapWord* high, HeapWord* low); + inline void set_offset_array(u_char* addr, HeapWord* high, HeapWord* low); - inline void set_offset_array(size_t left, size_t right, u_char offset); + inline void set_offset_array(u_char* left, u_char* right, u_char offset); - void check_index(size_t index, const char* msg) const NOT_DEBUG_RETURN; + void check_address(u_char* addr, const char* msg) const NOT_DEBUG_RETURN; public: @@ -92,17 +91,11 @@ public: // in the heap parameter. G1BlockOffsetTable(MemRegion heap, G1RegionToSpaceMapper* storage); - // Return the appropriate index into "_offset_array" for "p". - inline size_t index_for(const void* p) const; - inline size_t index_for_raw(const void* p) const; + // Mapping from address to object start array entry + u_char* entry_for_addr(const void* const p) const; - // Return the address indicating the start of the region corresponding to - // "index" in "_offset_array". - inline HeapWord* address_for_index(size_t index) const; - // Variant of address_for_index that does not check the index for validity. - inline HeapWord* address_for_index_raw(size_t index) const { - return _reserved.start() + (index << CardTable::card_shift_in_words()); - } + // Mapping from object start array entry to address of first word + HeapWord* addr_for_entry(const u_char* const p) const; }; class G1BlockOffsetTablePart { @@ -117,12 +110,12 @@ private: // Sets the entries corresponding to the cards starting at "start" and ending // at "end" to point back to the card before "start"; [start, end] - void set_remainder_to_point_to_start_incl(size_t start, size_t end); + void set_remainder_to_point_to_start_incl(u_char* start, u_char* end); // Update BOT entries corresponding to the mem range [blk_start, blk_end). void update_for_block_work(HeapWord* blk_start, HeapWord* blk_end); - void check_all_cards(size_t left_card, size_t right_card) const NOT_DEBUG_RETURN; + void check_all_cards(u_char* left_card, u_char* right_card) const NOT_DEBUG_RETURN; static HeapWord* align_up_by_card_size(HeapWord* const addr) { return align_up(addr, CardTable::card_size()); diff --git a/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp b/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp index 9625e0a5a57..7c665287ebf 100644 --- a/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp +++ b/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp @@ -38,81 +38,68 @@ inline HeapWord* G1BlockOffsetTablePart::block_start_reaching_into_card(const vo #ifdef ASSERT if (!_hr->is_continues_humongous()) { // For non-ContinuesHumongous regions, the first obj always starts from bottom. - u_char offset = _bot->offset_array(_bot->index_for(_hr->bottom())); + u_char offset = _bot->offset_array(_bot->entry_for_addr(_hr->bottom())); assert(offset == 0, "Found offset %u instead of 0 for region %u %s", offset, _hr->hrm_index(), _hr->get_short_type_str()); } #endif - size_t index = _bot->index_for(addr); - - u_char offset = _bot->offset_array(index); + u_char* entry = _bot->entry_for_addr(addr); + u_char offset = _bot->offset_array(entry); while (offset >= CardTable::card_size_in_words()) { // The excess of the offset from N_words indicates a power of Base // to go back by. size_t n_cards_back = BOTConstants::entry_to_cards_back(offset); - index -= n_cards_back; - offset = _bot->offset_array(index); + entry -= n_cards_back; + offset = _bot->offset_array(entry); } assert(offset < CardTable::card_size_in_words(), "offset too large"); - - HeapWord* q = _bot->address_for_index(index); + HeapWord* q = _bot->addr_for_entry(entry); return q - offset; } -u_char G1BlockOffsetTable::offset_array(size_t index) const { - check_index(index, "index out of range"); - return Atomic::load(&_offset_array[index]); +u_char G1BlockOffsetTable::offset_array(u_char* addr) const { + check_address(addr, "Block offset table address out of range"); + return Atomic::load(addr); } -void G1BlockOffsetTable::set_offset_array_raw(size_t index, u_char offset) { - Atomic::store(&_offset_array[index], offset); +void G1BlockOffsetTable::set_offset_array_raw(u_char* addr, u_char offset) { + Atomic::store(addr, offset); } -void G1BlockOffsetTable::set_offset_array(size_t index, u_char offset) { - check_index(index, "index out of range"); - set_offset_array_raw(index, offset); +void G1BlockOffsetTable::set_offset_array(u_char* addr, u_char offset) { + check_address(addr, "Block offset table address out of range"); + set_offset_array_raw(addr, offset); } -void G1BlockOffsetTable::set_offset_array(size_t index, HeapWord* high, HeapWord* low) { - check_index(index, "index out of range"); +void G1BlockOffsetTable::set_offset_array(u_char* addr, HeapWord* high, HeapWord* low) { + check_address(addr, "Block offset table address out of range"); assert(high >= low, "addresses out of order"); size_t offset = pointer_delta(high, low); check_offset(offset, "offset too large"); - set_offset_array(index, (u_char)offset); + set_offset_array(addr, (u_char)offset); } -void G1BlockOffsetTable::set_offset_array(size_t left, size_t right, u_char offset) { - check_index(right, "right index out of range"); +void G1BlockOffsetTable::set_offset_array(u_char* left, u_char* right, u_char offset) { + check_address(right, "Right block offset table address out of range"); assert(left <= right, "indexes out of order"); size_t num_cards = right - left + 1; memset_with_concurrent_readers - (const_cast (&_offset_array[left]), offset, num_cards); + (const_cast (left), offset, num_cards); } -// Variant of index_for that does not check the index for validity. -inline size_t G1BlockOffsetTable::index_for_raw(const void* p) const { - return pointer_delta((char*)p, _reserved.start(), sizeof(char)) >> CardTable::card_shift(); -} - -inline size_t G1BlockOffsetTable::index_for(const void* p) const { - char* pc = (char*)p; - assert(pc >= (char*)_reserved.start() && - pc < (char*)_reserved.end(), - "p (" PTR_FORMAT ") not in reserved [" PTR_FORMAT ", " PTR_FORMAT ")", - p2i(p), p2i(_reserved.start()), p2i(_reserved.end())); - size_t result = index_for_raw(p); - check_index(result, "bad index from address"); +inline u_char* G1BlockOffsetTable::entry_for_addr(const void* const p) const { + assert(_reserved.contains(p), + "out of bounds access to block offset table"); + u_char* result = const_cast(&_offset_base[uintptr_t(p) >> CardTable::card_shift()]); return result; } -inline HeapWord* G1BlockOffsetTable::address_for_index(size_t index) const { - check_index(index, "index out of range"); - HeapWord* result = address_for_index_raw(index); - assert(result >= _reserved.start() && result < _reserved.end(), - "bad address from index result " PTR_FORMAT - " _reserved.start() " PTR_FORMAT " _reserved.end() " PTR_FORMAT, - p2i(result), p2i(_reserved.start()), p2i(_reserved.end())); +inline HeapWord* G1BlockOffsetTable::addr_for_entry(const u_char* const p) const { + size_t delta = pointer_delta(p, _offset_base, sizeof(u_char)); + HeapWord* result = (HeapWord*) (delta << CardTable::card_shift()); + assert(_reserved.contains(result), + "out of bounds accessor from block offset table"); return result; } diff --git a/src/hotspot/share/gc/g1/g1RemSet.cpp b/src/hotspot/share/gc/g1/g1RemSet.cpp index c5cb686d132..2b338d89e04 100644 --- a/src/hotspot/share/gc/g1/g1RemSet.cpp +++ b/src/hotspot/share/gc/g1/g1RemSet.cpp @@ -531,7 +531,7 @@ class G1ScanHRForRegionClosure : public HeapRegionClosure { return; } - HeapWord* scan_end = MIN2(card_start + (num_cards << CardTable::card_shift_in_words()), top); + HeapWord* scan_end = MIN2(card_start + (num_cards << (CardTable::card_shift() - LogHeapWordSize)), top); if (_scanned_to >= scan_end) { return; } diff --git a/src/hotspot/share/gc/serial/serialBlockOffsetTable.cpp b/src/hotspot/share/gc/serial/serialBlockOffsetTable.cpp index d49669fa5ff..95a1c451fbb 100644 --- a/src/hotspot/share/gc/serial/serialBlockOffsetTable.cpp +++ b/src/hotspot/share/gc/serial/serialBlockOffsetTable.cpp @@ -47,7 +47,7 @@ SerialBlockOffsetSharedArray::SerialBlockOffsetSharedArray(MemRegion reserved, if (!_vs.initialize(rs, 0)) { vm_exit_during_initialization("Could not reserve enough space for heap offset array"); } - _offset_array = (uint8_t*)_vs.low_boundary(); + _offset_base = (uint8_t*)(_vs.low_boundary() - (uintptr_t(reserved.start()) >> CardTable::card_shift())); resize(init_word_size); log_trace(gc, bot)("SerialBlockOffsetSharedArray::SerialBlockOffsetSharedArray: "); log_trace(gc, bot)(" rs.base(): " PTR_FORMAT " rs.size(): " SIZE_FORMAT_X_0 " rs end(): " PTR_FORMAT, @@ -102,7 +102,6 @@ void SerialBlockOffsetSharedArray::resize(size_t new_word_size) { // the index of the entry // // Given an address, -// Find the index for the address // Find the block offset table entry // Convert the entry to a back slide // (e.g., with today's, offset = 0x81 => @@ -113,22 +112,22 @@ void SerialBlockOffsetSharedArray::resize(size_t new_word_size) { void SerialBlockOffsetTable::update_for_block_work(HeapWord* blk_start, HeapWord* blk_end) { HeapWord* const cur_card_boundary = align_up_by_card_size(blk_start); - size_t const offset_card = _array->index_for(cur_card_boundary); + uint8_t* const offset_card = _array->entry_for_addr(cur_card_boundary); // The first card holds the actual offset. _array->set_offset_array(offset_card, cur_card_boundary, blk_start); // Check if this block spans over other cards. - size_t end_card = _array->index_for(blk_end - 1); + uint8_t* end_card = _array->entry_for_addr(blk_end - 1); assert(offset_card <= end_card, "inv"); if (offset_card != end_card) { // Handling remaining cards. - size_t start_card_for_region = offset_card + 1; + uint8_t* start_card_for_region = offset_card + 1; for (uint i = 0; i < BOTConstants::N_powers; i++) { // -1 so that the reach ends in this region and not at the start // of the next. - size_t reach = offset_card + BOTConstants::power_to_cards_back(i + 1) - 1; + uint8_t* reach = offset_card + BOTConstants::power_to_cards_back(i + 1) - 1; uint8_t value = checked_cast(CardTable::card_size_in_words() + i); _array->set_offset_array(start_card_for_region, MIN2(reach, end_card), value); @@ -145,40 +144,33 @@ void SerialBlockOffsetTable::update_for_block_work(HeapWord* blk_start, } HeapWord* SerialBlockOffsetTable::block_start_reaching_into_card(const void* addr) const { - size_t index = _array->index_for(addr); - - uint8_t offset; - while (true) { - offset = _array->offset_array(index); - - if (offset < CardTable::card_size_in_words()) { - break; - } - + uint8_t* entry = _array->entry_for_addr(addr); + uint8_t offset = *entry; + while (offset >= CardTable::card_size_in_words()) { // The excess of the offset from N_words indicates a power of Base // to go back by. size_t n_cards_back = BOTConstants::entry_to_cards_back(offset); - index -= n_cards_back; + entry -= n_cards_back; + offset = *entry; } - - HeapWord* q = _array->address_for_index(index); + HeapWord* q = _array->addr_for_entry(entry); return q - offset; } void SerialBlockOffsetTable::verify_for_block(HeapWord* blk_start, HeapWord* blk_end) const { assert(is_crossing_card_boundary(blk_start, blk_end), "precondition"); - const size_t start_card = _array->index_for(align_up_by_card_size(blk_start)); - const size_t end_card = _array->index_for(blk_end - 1); + uint8_t* start_card = _array->entry_for_addr(align_up_by_card_size(blk_start)); + uint8_t* end_card = _array->entry_for_addr(blk_end - 1); // Check cards in [start_card, end_card] - assert(_array->offset_array(start_card) < CardTable::card_size_in_words(), "offset card"); + assert(*start_card < CardTable::card_size_in_words(), "offset card"); - for (size_t i = start_card + 1; i <= end_card; ++i) { - const uint8_t prev = _array->offset_array(i-1); - const uint8_t value = _array->offset_array(i); - if (prev != value) { - assert(value >= prev, "monotonic"); - size_t n_cards_back = BOTConstants::entry_to_cards_back(value); + for (uint8_t* i = start_card + 1; i <= end_card; ++i) { + const uint8_t* prev = i - 1; + const uint8_t* value = i; + if (*prev != *value) { + assert(*value >= *prev, "monotonic"); + size_t n_cards_back = BOTConstants::entry_to_cards_back(*value); assert(start_card == (i - n_cards_back), "inv"); } } diff --git a/src/hotspot/share/gc/serial/serialBlockOffsetTable.hpp b/src/hotspot/share/gc/serial/serialBlockOffsetTable.hpp index fa422a96284..4765c36908d 100644 --- a/src/hotspot/share/gc/serial/serialBlockOffsetTable.hpp +++ b/src/hotspot/share/gc/serial/serialBlockOffsetTable.hpp @@ -46,16 +46,12 @@ class SerialBlockOffsetSharedArray: public CHeapObj { // Array for keeping offsets for retrieving object start fast given an // address. VirtualSpace _vs; - uint8_t* _offset_array; // byte array keeping backwards offsets - void fill_range(size_t start, size_t num_cards, uint8_t offset) { - void* start_ptr = &_offset_array[start]; - memset(start_ptr, offset, num_cards); - } + // Biased array-start of BOT array for fast BOT entry translation + uint8_t* _offset_base; - uint8_t offset_array(size_t index) const { - assert(index < _vs.committed_size(), "index out of range"); - return _offset_array[index]; + void fill_range(uint8_t* const start, size_t num_cards, uint8_t offset) { + memset(start, offset, num_cards); } // Return the number of slots needed for an offset array @@ -81,22 +77,21 @@ public: // reserved region this table covers. void resize(size_t new_word_size); - // Return the appropriate index into "_offset_array" for "p". - size_t index_for(const void* p) const; + // Mapping from address to object start array entry + uint8_t* entry_for_addr(const void* const p) const; - // Return the address indicating the start of the region corresponding to - // "index" in "_offset_array". - HeapWord* address_for_index(size_t index) const; + // Mapping from object start array entry to address of first word + HeapWord* addr_for_entry(const uint8_t* const p) const; - void set_offset_array(size_t index, HeapWord* high, HeapWord* low) { - assert(index < _vs.committed_size(), "index out of range"); + void set_offset_array(uint8_t* const addr, HeapWord* high, HeapWord* low) { + assert(_vs.contains(addr), "Block offset address out of range"); assert(high >= low, "addresses out of order"); assert(pointer_delta(high, low) < CardTable::card_size_in_words(), "offset too large"); - _offset_array[index] = checked_cast(pointer_delta(high, low)); + *addr = checked_cast(pointer_delta(high, low)); } - void set_offset_array(size_t left, size_t right, uint8_t offset) { - assert(right < _vs.committed_size(), "right address out of range"); + void set_offset_array(uint8_t* const left, uint8_t* const right, uint8_t offset) { + assert(_vs.contains(right), "right address out of range"); assert(left <= right, "precondition"); size_t num_cards = right - left + 1; diff --git a/src/hotspot/share/gc/serial/serialBlockOffsetTable.inline.hpp b/src/hotspot/share/gc/serial/serialBlockOffsetTable.inline.hpp index b9378de2b75..5ca0a73ac65 100644 --- a/src/hotspot/share/gc/serial/serialBlockOffsetTable.inline.hpp +++ b/src/hotspot/share/gc/serial/serialBlockOffsetTable.inline.hpp @@ -27,22 +27,18 @@ #include "gc/serial/serialBlockOffsetTable.hpp" -inline size_t SerialBlockOffsetSharedArray::index_for(const void* p) const { - char* pc = (char*)p; - assert(pc >= (char*)_reserved.start() && - pc < (char*)_reserved.end(), - "p not in range."); - size_t delta = pointer_delta(pc, _reserved.start(), sizeof(char)); - size_t result = delta >> CardTable::card_shift(); - assert(result < _vs.committed_size(), "bad index from address"); +inline uint8_t* SerialBlockOffsetSharedArray::entry_for_addr(const void* const p) const { + assert(_reserved.contains(p), + "out of bounds access to block offset array"); + uint8_t* result = &_offset_base[uintptr_t(p) >> CardTable::card_shift()]; return result; } -inline HeapWord* SerialBlockOffsetSharedArray::address_for_index(size_t index) const { - assert(index < _vs.committed_size(), "bad index"); - HeapWord* result = _reserved.start() + (index << CardTable::card_shift_in_words()); - assert(result >= _reserved.start() && result < _reserved.end(), - "bad address from index"); +inline HeapWord* SerialBlockOffsetSharedArray::addr_for_entry(const uint8_t* const p) const { + size_t delta = pointer_delta(p, _offset_base, sizeof(uint8_t)); + HeapWord* result = (HeapWord*) (delta << CardTable::card_shift()); + assert(_reserved.contains(result), + "out of bounds accessor from block offset array"); return result; } diff --git a/src/hotspot/share/gc/serial/vmStructs_serial.hpp b/src/hotspot/share/gc/serial/vmStructs_serial.hpp index 9f646424100..bd2ff4b109a 100644 --- a/src/hotspot/share/gc/serial/vmStructs_serial.hpp +++ b/src/hotspot/share/gc/serial/vmStructs_serial.hpp @@ -58,7 +58,7 @@ \ nonstatic_field(SerialBlockOffsetSharedArray, _reserved, MemRegion) \ nonstatic_field(SerialBlockOffsetSharedArray, _vs, VirtualSpace) \ - nonstatic_field(SerialBlockOffsetSharedArray, _offset_array, u_char*) \ + nonstatic_field(SerialBlockOffsetSharedArray, _offset_base, u_char*) \ \ nonstatic_field(TenuredSpace, _offsets, SerialBlockOffsetTable) \ \ diff --git a/src/hotspot/share/gc/shared/cardTable.cpp b/src/hotspot/share/gc/shared/cardTable.cpp index a1d8571fa93..b4accf61a99 100644 --- a/src/hotspot/share/gc/shared/cardTable.cpp +++ b/src/hotspot/share/gc/shared/cardTable.cpp @@ -40,7 +40,6 @@ #endif uint CardTable::_card_shift = 0; -uint CardTable::_card_shift_in_words = 0; uint CardTable::_card_size = 0; uint CardTable::_card_size_in_words = 0; @@ -51,7 +50,6 @@ void CardTable::initialize_card_size() { _card_size = GCCardSizeInBytes; _card_shift = log2i_exact(_card_size); _card_size_in_words = _card_size / sizeof(HeapWord); - _card_shift_in_words = _card_shift - LogHeapWordSize; log_info_p(gc, init)("CardTable entry size: " UINT32_FORMAT, _card_size); } diff --git a/src/hotspot/share/gc/shared/cardTable.hpp b/src/hotspot/share/gc/shared/cardTable.hpp index f8441d3342a..13285be4fc5 100644 --- a/src/hotspot/share/gc/shared/cardTable.hpp +++ b/src/hotspot/share/gc/shared/cardTable.hpp @@ -73,7 +73,6 @@ protected: // CardTable entry size static uint _card_shift; - static uint _card_shift_in_words; static uint _card_size; static uint _card_size_in_words; @@ -183,10 +182,6 @@ public: return _card_shift; } - static uint card_shift_in_words() { - return _card_shift_in_words; - } - static uint card_size() { return _card_size; }