infernap12/jdk
1
0
Fork 0
mirror of https://github.com/openjdk/jdk.git synced 2026-01-28 03:58:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

8375436: G1: Convert G1CardSet classes to use Atomic<T>

Browse Source 
Reviewed-by: kbarrett, iwalulya
This commit is contained in:
Thomas Schatzl 2026-01-26 09:12:39 +00:00
parent 90b5469253
commit 2af271e5e6
6 changed files with 115 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
src/hotspot/share/gc/g1/g1CardSet.cpp
View File

@ -29,7 +29,6 @@
#include "gc/shared/gcLogPrecious.hpp" #include "gc/shared/gcLogPrecious.hpp"
#include "gc/shared/gcTraceTime.inline.hpp" #include "gc/shared/gcTraceTime.inline.hpp"
#include "memory/allocation.inline.hpp" #include "memory/allocation.inline.hpp"
#include "runtime/atomicAccess.hpp"
#include "runtime/globals_extension.hpp" #include "runtime/globals_extension.hpp"
#include "runtime/java.hpp" #include "runtime/java.hpp"
#include "utilities/bitMap.inline.hpp" #include "utilities/bitMap.inline.hpp"
@ -192,32 +191,32 @@ const char* G1CardSetConfiguration::mem_object_type_name_str(uint index) {
void G1CardSetCoarsenStats::reset() { void G1CardSetCoarsenStats::reset() {
STATIC_ASSERT(ARRAY_SIZE(_coarsen_from) == ARRAY_SIZE(_coarsen_collision)); STATIC_ASSERT(ARRAY_SIZE(_coarsen_from) == ARRAY_SIZE(_coarsen_collision));
for (uint i = 0; i < ARRAY_SIZE(_coarsen_from); i++) { for (uint i = 0; i < ARRAY_SIZE(_coarsen_from); i++) {
_coarsen_from[i] = 0; _coarsen_from[i].store_relaxed(0);
_coarsen_collision[i] = 0; _coarsen_collision[i].store_relaxed(0);
} }
} }
void G1CardSetCoarsenStats::set(G1CardSetCoarsenStats& other) { void G1CardSetCoarsenStats::set(G1CardSetCoarsenStats& other) {
STATIC_ASSERT(ARRAY_SIZE(_coarsen_from) == ARRAY_SIZE(_coarsen_collision)); STATIC_ASSERT(ARRAY_SIZE(_coarsen_from) == ARRAY_SIZE(_coarsen_collision));
for (uint i = 0; i < ARRAY_SIZE(_coarsen_from); i++) { for (uint i = 0; i < ARRAY_SIZE(_coarsen_from); i++) {
_coarsen_from[i] = other._coarsen_from[i]; _coarsen_from[i].store_relaxed(other._coarsen_from[i].load_relaxed());
_coarsen_collision[i] = other._coarsen_collision[i]; _coarsen_collision[i].store_relaxed(other._coarsen_collision[i].load_relaxed());
} }
} }
void G1CardSetCoarsenStats::subtract_from(G1CardSetCoarsenStats& other) { void G1CardSetCoarsenStats::subtract_from(G1CardSetCoarsenStats& other) {
STATIC_ASSERT(ARRAY_SIZE(_coarsen_from) == ARRAY_SIZE(_coarsen_collision)); STATIC_ASSERT(ARRAY_SIZE(_coarsen_from) == ARRAY_SIZE(_coarsen_collision));
for (uint i = 0; i < ARRAY_SIZE(_coarsen_from); i++) { for (uint i = 0; i < ARRAY_SIZE(_coarsen_from); i++) {
_coarsen_from[i] = other._coarsen_from[i] - _coarsen_from[i]; _coarsen_from[i].store_relaxed(other._coarsen_from[i].load_relaxed() - _coarsen_from[i].load_relaxed());
_coarsen_collision[i] = other._coarsen_collision[i] - _coarsen_collision[i]; _coarsen_collision[i].store_relaxed(other._coarsen_collision[i].load_relaxed() - _coarsen_collision[i].load_relaxed());
} }
} }
void G1CardSetCoarsenStats::record_coarsening(uint tag, bool collision) { void G1CardSetCoarsenStats::record_coarsening(uint tag, bool collision) {
assert(tag < ARRAY_SIZE(_coarsen_from), "tag %u out of bounds", tag); assert(tag < ARRAY_SIZE(_coarsen_from), "tag %u out of bounds", tag);
AtomicAccess::inc(&_coarsen_from[tag], memory_order_relaxed); _coarsen_from[tag].add_then_fetch(1u, memory_order_relaxed);
if (collision) { if (collision) {
AtomicAccess::inc(&_coarsen_collision[tag], memory_order_relaxed); _coarsen_collision[tag].add_then_fetch(1u, memory_order_relaxed);
} }
} }
@ -228,13 +227,13 @@ void G1CardSetCoarsenStats::print_on(outputStream* out) {
"Inline->AoC %zu (%zu) " "Inline->AoC %zu (%zu) "
"AoC->BitMap %zu (%zu) " "AoC->BitMap %zu (%zu) "
"BitMap->Full %zu (%zu) ", "BitMap->Full %zu (%zu) ",
_coarsen_from[0], _coarsen_collision[0], _coarsen_from[0].load_relaxed(), _coarsen_collision[0].load_relaxed(),
_coarsen_from[1], _coarsen_collision[1], _coarsen_from[1].load_relaxed(), _coarsen_collision[1].load_relaxed(),
// There is no BitMap at the first level so we can't . // There is no BitMap at the first level so we can't .
_coarsen_from[3], _coarsen_collision[3], _coarsen_from[3].load_relaxed(), _coarsen_collision[3].load_relaxed(),
_coarsen_from[4], _coarsen_collision[4], _coarsen_from[4].load_relaxed(), _coarsen_collision[4].load_relaxed(),
_coarsen_from[5], _coarsen_collision[5], _coarsen_from[5].load_relaxed(), _coarsen_collision[5].load_relaxed(),
_coarsen_from[6], _coarsen_collision[6] _coarsen_from[6].load_relaxed(), _coarsen_collision[6].load_relaxed()
); );
} }
@ -248,7 +247,7 @@ class G1CardSetHashTable : public CHeapObj<mtGCCardSet> {
// the per region cardsets. // the per region cardsets.
const static uint GroupBucketClaimSize = 4; const static uint GroupBucketClaimSize = 4;
// Did we insert at least one card in the table? // Did we insert at least one card in the table?
bool volatile _inserted_card; Atomic<bool> _inserted_card;
G1CardSetMemoryManager* _mm; G1CardSetMemoryManager* _mm;
CardSetHash _table; CardSetHash _table;
@ -311,10 +310,10 @@ public:
G1CardSetHashTableValue value(region_idx, G1CardSetInlinePtr()); G1CardSetHashTableValue value(region_idx, G1CardSetInlinePtr());
bool inserted = _table.insert_get(Thread::current(), lookup, value, found, should_grow); bool inserted = _table.insert_get(Thread::current(), lookup, value, found, should_grow);
if (!_inserted_card && inserted) { if (!_inserted_card.load_relaxed() && inserted) {
// It does not matter to us who is setting the flag so a regular atomic store // It does not matter to us who is setting the flag so a regular atomic store
// is sufficient. // is sufficient.
AtomicAccess::store(&_inserted_card, true); _inserted_card.store_relaxed(true);
} }
return found.value(); return found.value();
@ -343,9 +342,9 @@ public:
} }
void reset() { void reset() {
if (AtomicAccess::load(&_inserted_card)) { if (_inserted_card.load_relaxed()) {
_table.unsafe_reset(InitialLogTableSize); _table.unsafe_reset(InitialLogTableSize);
AtomicAccess::store(&_inserted_card, false); _inserted_card.store_relaxed(false);
} }
} }
@ -455,14 +454,14 @@ void G1CardSet::free_mem_object(ContainerPtr container) {
_mm->free(container_type_to_mem_object_type(type), value); _mm->free(container_type_to_mem_object_type(type), value);
} }
G1CardSet::ContainerPtr G1CardSet::acquire_container(ContainerPtr volatile* container_addr) { G1CardSet::ContainerPtr G1CardSet::acquire_container(Atomic<ContainerPtr>* container_addr) {
// Update reference counts under RCU critical section to avoid a // Update reference counts under RCU critical section to avoid a
// use-after-cleapup bug where we increment a reference count for // use-after-cleapup bug where we increment a reference count for
// an object whose memory has already been cleaned up and reused. // an object whose memory has already been cleaned up and reused.
GlobalCounter::CriticalSection cs(Thread::current()); GlobalCounter::CriticalSection cs(Thread::current());
while (true) { while (true) {
// Get ContainerPtr and increment refcount atomically wrt to memory reuse. // Get ContainerPtr and increment refcount atomically wrt to memory reuse.
ContainerPtr container = AtomicAccess::load_acquire(container_addr); ContainerPtr container = container_addr->load_acquire();
uint cs_type = container_type(container); uint cs_type = container_type(container);
if (container == FullCardSet || cs_type == ContainerInlinePtr) { if (container == FullCardSet || cs_type == ContainerInlinePtr) {
return container; return container;
@ -503,15 +502,15 @@ class G1ReleaseCardsets : public StackObj {
G1CardSet* _card_set; G1CardSet* _card_set;
using ContainerPtr = G1CardSet::ContainerPtr; using ContainerPtr = G1CardSet::ContainerPtr;
void coarsen_to_full(ContainerPtr* container_addr) { void coarsen_to_full(Atomic<ContainerPtr>* container_addr) {
while (true) { while (true) {
ContainerPtr cur_container = AtomicAccess::load_acquire(container_addr); ContainerPtr cur_container = container_addr->load_acquire();
uint cs_type = G1CardSet::container_type(cur_container); uint cs_type = G1CardSet::container_type(cur_container);
if (cur_container == G1CardSet::FullCardSet) { if (cur_container == G1CardSet::FullCardSet) {
return; return;
} }
ContainerPtr old_value = AtomicAccess::cmpxchg(container_addr, cur_container, G1CardSet::FullCardSet); ContainerPtr old_value = container_addr->compare_exchange(cur_container, G1CardSet::FullCardSet);
if (old_value == cur_container) { if (old_value == cur_container) {
_card_set->release_and_maybe_free_container(cur_container); _card_set->release_and_maybe_free_container(cur_container);
@ -523,7 +522,7 @@ class G1ReleaseCardsets : public StackObj {
public: public:
explicit G1ReleaseCardsets(G1CardSet* card_set) : _card_set(card_set) { } explicit G1ReleaseCardsets(G1CardSet* card_set) : _card_set(card_set) { }
void operator ()(ContainerPtr* container_addr) { void operator ()(Atomic<ContainerPtr>* container_addr) {
coarsen_to_full(container_addr); coarsen_to_full(container_addr);
} }
}; };
@ -544,10 +543,10 @@ G1AddCardResult G1CardSet::add_to_howl(ContainerPtr parent_container,
ContainerPtr container; ContainerPtr container;
uint bucket = _config->howl_bucket_index(card_in_region); uint bucket = _config->howl_bucket_index(card_in_region);
ContainerPtr volatile* bucket_entry = howl->container_addr(bucket); Atomic<ContainerPtr>* bucket_entry = howl->container_addr(bucket);
while (true) { while (true) {
if (AtomicAccess::load(&howl->_num_entries) >= _config->cards_in_howl_threshold()) { if (howl->_num_entries.load_relaxed() >= _config->cards_in_howl_threshold()) {
return Overflow; return Overflow;
} }
@ -571,7 +570,7 @@ G1AddCardResult G1CardSet::add_to_howl(ContainerPtr parent_container,
} }
if (increment_total && add_result == Added) { if (increment_total && add_result == Added) {
AtomicAccess::inc(&howl->_num_entries, memory_order_relaxed); howl->_num_entries.add_then_fetch(1u, memory_order_relaxed);
} }
if (to_transfer != nullptr) { if (to_transfer != nullptr) {
@ -588,7 +587,7 @@ G1AddCardResult G1CardSet::add_to_bitmap(ContainerPtr container, uint card_in_re
return bitmap->add(card_offset, _config->cards_in_howl_bitmap_threshold(), _config->max_cards_in_howl_bitmap()); return bitmap->add(card_offset, _config->cards_in_howl_bitmap_threshold(), _config->max_cards_in_howl_bitmap());
} }
G1AddCardResult G1CardSet::add_to_inline_ptr(ContainerPtr volatile* container_addr, ContainerPtr container, uint card_in_region) { G1AddCardResult G1CardSet::add_to_inline_ptr(Atomic<ContainerPtr>* container_addr, ContainerPtr container, uint card_in_region) {
G1CardSetInlinePtr value(container_addr, container); G1CardSetInlinePtr value(container_addr, container);
return value.add(card_in_region, _config->inline_ptr_bits_per_card(), _config->max_cards_in_inline_ptr()); return value.add(card_in_region, _config->inline_ptr_bits_per_card(), _config->max_cards_in_inline_ptr());
} }
@ -610,7 +609,7 @@ G1CardSet::ContainerPtr G1CardSet::create_coarsened_array_of_cards(uint card_in_
return new_container; return new_container;
} }
bool G1CardSet::coarsen_container(ContainerPtr volatile* container_addr, bool G1CardSet::coarsen_container(Atomic<ContainerPtr>* container_addr,
ContainerPtr cur_container, ContainerPtr cur_container,
uint card_in_region, uint card_in_region,
bool within_howl) { bool within_howl) {
@ -640,7 +639,7 @@ bool G1CardSet::coarsen_container(ContainerPtr volatile* container_addr,
ShouldNotReachHere(); ShouldNotReachHere();
} }
ContainerPtr old_value = AtomicAccess::cmpxchg(container_addr, cur_container, new_container); // Memory order? ContainerPtr old_value = container_addr->compare_exchange(cur_container, new_container); // Memory order?
if (old_value == cur_container) { if (old_value == cur_container) {
// Success. Indicate that the cards from the current card set must be transferred // Success. Indicate that the cards from the current card set must be transferred
// by this caller. // by this caller.
@ -687,7 +686,7 @@ void G1CardSet::transfer_cards(G1CardSetHashTableValue* table_entry, ContainerPt
assert(container_type(source_container) == ContainerHowl, "must be"); assert(container_type(source_container) == ContainerHowl, "must be");
// Need to correct for that the Full remembered set occupies more cards than the // Need to correct for that the Full remembered set occupies more cards than the
// AoCS before. // AoCS before.
AtomicAccess::add(&_num_occupied, _config->max_cards_in_region() - table_entry->_num_occupied, memory_order_relaxed); _num_occupied.add_then_fetch(_config->max_cards_in_region() - table_entry->_num_occupied.load_relaxed(), memory_order_relaxed);
} }
} }
@ -713,18 +712,18 @@ void G1CardSet::transfer_cards_in_howl(ContainerPtr parent_container,
diff -= 1; diff -= 1;
G1CardSetHowl* howling_array = container_ptr<G1CardSetHowl>(parent_container); G1CardSetHowl* howling_array = container_ptr<G1CardSetHowl>(parent_container);
AtomicAccess::add(&howling_array->_num_entries, diff, memory_order_relaxed); howling_array->_num_entries.add_then_fetch(diff, memory_order_relaxed);
G1CardSetHashTableValue* table_entry = get_container(card_region); G1CardSetHashTableValue* table_entry = get_container(card_region);
assert(table_entry != nullptr, "Table entry not found for transferred cards"); assert(table_entry != nullptr, "Table entry not found for transferred cards");
AtomicAccess::add(&table_entry->_num_occupied, diff, memory_order_relaxed); table_entry->_num_occupied.add_then_fetch(diff, memory_order_relaxed);
AtomicAccess::add(&_num_occupied, diff, memory_order_relaxed); _num_occupied.add_then_fetch(diff, memory_order_relaxed);
} }
} }
G1AddCardResult G1CardSet::add_to_container(ContainerPtr volatile* container_addr, G1AddCardResult G1CardSet::add_to_container(Atomic<ContainerPtr>* container_addr,
ContainerPtr container, ContainerPtr container,
uint card_region, uint card_region,
uint card_in_region, uint card_in_region,
@ -827,8 +826,8 @@ G1AddCardResult G1CardSet::add_card(uint card_region, uint card_in_region, bool
} }
if (increment_total && add_result == Added) { if (increment_total && add_result == Added) {
AtomicAccess::inc(&table_entry->_num_occupied, memory_order_relaxed); table_entry->_num_occupied.add_then_fetch(1u, memory_order_relaxed);
AtomicAccess::inc(&_num_occupied, memory_order_relaxed); _num_occupied.add_then_fetch(1u, memory_order_relaxed);
} }
if (should_grow_table) { if (should_grow_table) {
_table->grow(); _table->grow();
@ -853,7 +852,7 @@ bool G1CardSet::contains_card(uint card_region, uint card_in_region) {
return false; return false;
} }
ContainerPtr container = table_entry->_container; ContainerPtr container = table_entry->_container.load_relaxed();
if (container == FullCardSet) { if (container == FullCardSet) {
// contains_card() is not a performance critical method so we do not hide that // contains_card() is not a performance critical method so we do not hide that
// case in the switch below. // case in the switch below.
@ -889,7 +888,7 @@ void G1CardSet::print_info(outputStream* st, uintptr_t card) {
return; return;
} }
ContainerPtr container = table_entry->_container; ContainerPtr container = table_entry->_container.load_relaxed();
if (container == FullCardSet) { if (container == FullCardSet) {
st->print("FULL card set)"); st->print("FULL card set)");
return; return;
@ -940,7 +939,7 @@ void G1CardSet::iterate_cards_during_transfer(ContainerPtr const container, Card
void G1CardSet::iterate_containers(ContainerPtrClosure* cl, bool at_safepoint) { void G1CardSet::iterate_containers(ContainerPtrClosure* cl, bool at_safepoint) {
auto do_value = auto do_value =
[&] (G1CardSetHashTableValue* value) { [&] (G1CardSetHashTableValue* value) {
cl->do_containerptr(value->_region_idx, value->_num_occupied, value->_container); cl->do_containerptr(value->_region_idx, value->_num_occupied.load_relaxed(), value->_container.load_relaxed());
return true; return true;
}; };
@ -1001,11 +1000,11 @@ bool G1CardSet::occupancy_less_or_equal_to(size_t limit) const {
} }
bool G1CardSet::is_empty() const { bool G1CardSet::is_empty() const {
return _num_occupied == 0; return _num_occupied.load_relaxed() == 0;
} }
size_t G1CardSet::occupied() const { size_t G1CardSet::occupied() const {
return _num_occupied; return _num_occupied.load_relaxed();
} }
size_t G1CardSet::num_containers() { size_t G1CardSet::num_containers() {
@ -1051,7 +1050,7 @@ size_t G1CardSet::static_mem_size() {
void G1CardSet::clear() { void G1CardSet::clear() {
_table->reset(); _table->reset();
_num_occupied = 0; _num_occupied.store_relaxed(0);
_mm->flush(); _mm->flush();
} }

29
src/hotspot/share/gc/g1/g1CardSet.hpp
View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
* *
* This code is free software; you can redistribute it and/or modify it * This code is free software; you can redistribute it and/or modify it
@ -27,6 +27,7 @@
#include "memory/allocation.hpp" #include "memory/allocation.hpp"
#include "memory/memRegion.hpp" #include "memory/memRegion.hpp"
#include "runtime/atomic.hpp"
#include "utilities/concurrentHashTable.hpp" #include "utilities/concurrentHashTable.hpp"
class G1CardSetAllocOptions; class G1CardSetAllocOptions;
@ -154,8 +155,8 @@ public:
private: private:
// Indices are "from" indices. // Indices are "from" indices.
size_t _coarsen_from[NumCoarsenCategories]; Atomic<size_t> _coarsen_from[NumCoarsenCategories];
size_t _coarsen_collision[NumCoarsenCategories]; Atomic<size_t> _coarsen_collision[NumCoarsenCategories];
public: public:
G1CardSetCoarsenStats() { reset(); } G1CardSetCoarsenStats() { reset(); }
@ -271,11 +272,11 @@ private:
// Total number of cards in this card set. This is a best-effort value, i.e. there may // Total number of cards in this card set. This is a best-effort value, i.e. there may
// be (slightly) more cards in the card set than this value in reality. // be (slightly) more cards in the card set than this value in reality.
size_t _num_occupied; Atomic<size_t> _num_occupied;
ContainerPtr make_container_ptr(void* value, uintptr_t type); ContainerPtr make_container_ptr(void* value, uintptr_t type);
ContainerPtr acquire_container(ContainerPtr volatile* container_addr); ContainerPtr acquire_container(Atomic<ContainerPtr>* container_addr);
// Returns true if the card set container should be released // Returns true if the card set container should be released
bool release_container(ContainerPtr container); bool release_container(ContainerPtr container);
// Release card set and free if needed. // Release card set and free if needed.
@ -288,7 +289,7 @@ private:
// coarsen_container does not transfer cards from cur_container // coarsen_container does not transfer cards from cur_container
// to the new container. Transfer is achieved by transfer_cards. // to the new container. Transfer is achieved by transfer_cards.
// Returns true if this was the thread that coarsened the container (and added the card). // Returns true if this was the thread that coarsened the container (and added the card).
bool coarsen_container(ContainerPtr volatile* container_addr, bool coarsen_container(Atomic<ContainerPtr>* container_addr,
ContainerPtr cur_container, ContainerPtr cur_container,
uint card_in_region, bool within_howl = false); uint card_in_region, bool within_howl = false);
@ -300,9 +301,9 @@ private:
void transfer_cards(G1CardSetHashTableValue* table_entry, ContainerPtr source_container, uint card_region); void transfer_cards(G1CardSetHashTableValue* table_entry, ContainerPtr source_container, uint card_region);
void transfer_cards_in_howl(ContainerPtr parent_container, ContainerPtr source_container, uint card_region); void transfer_cards_in_howl(ContainerPtr parent_container, ContainerPtr source_container, uint card_region);
G1AddCardResult add_to_container(ContainerPtr volatile* container_addr, ContainerPtr container, uint card_region, uint card, bool increment_total = true); G1AddCardResult add_to_container(Atomic<ContainerPtr>* container_addr, ContainerPtr container, uint card_region, uint card, bool increment_total = true);
G1AddCardResult add_to_inline_ptr(ContainerPtr volatile* container_addr, ContainerPtr container, uint card_in_region); G1AddCardResult add_to_inline_ptr(Atomic<ContainerPtr>* container_addr, ContainerPtr container, uint card_in_region);
G1AddCardResult add_to_array(ContainerPtr container, uint card_in_region); G1AddCardResult add_to_array(ContainerPtr container, uint card_in_region);
G1AddCardResult add_to_bitmap(ContainerPtr container, uint card_in_region); G1AddCardResult add_to_bitmap(ContainerPtr container, uint card_in_region);
G1AddCardResult add_to_howl(ContainerPtr parent_container, uint card_region, uint card_in_region, bool increment_total = true); G1AddCardResult add_to_howl(ContainerPtr parent_container, uint card_region, uint card_in_region, bool increment_total = true);
@ -366,7 +367,6 @@ public:
size_t num_containers(); size_t num_containers();
static G1CardSetCoarsenStats coarsen_stats();
static void print_coarsen_stats(outputStream* out); static void print_coarsen_stats(outputStream* out);
// Returns size of the actual remembered set containers in bytes. // Returns size of the actual remembered set containers in bytes.
@ -412,8 +412,15 @@ public:
using ContainerPtr = G1CardSet::ContainerPtr; using ContainerPtr = G1CardSet::ContainerPtr;
const uint _region_idx; const uint _region_idx;
uint volatile _num_occupied; Atomic<uint> _num_occupied;
ContainerPtr volatile _container; Atomic<ContainerPtr> _container;
// Copy constructor needed for use in ConcurrentHashTable.
G1CardSetHashTableValue(const G1CardSetHashTableValue& other) :
_region_idx(other._region_idx),
_num_occupied(other._num_occupied.load_relaxed()),
_container(other._container.load_relaxed())
{ }
G1CardSetHashTableValue(uint region_idx, ContainerPtr container) : _region_idx(region_idx), _num_occupied(0), _container(container) { } G1CardSetHashTableValue(uint region_idx, ContainerPtr container) : _region_idx(region_idx), _num_occupied(0), _container(container) { }
}; };

36
src/hotspot/share/gc/g1/g1CardSetContainers.hpp
View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2023, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
* *
* This code is free software; you can redistribute it and/or modify it * This code is free software; you can redistribute it and/or modify it
@ -27,7 +27,7 @@
#include "gc/g1/g1CardSet.hpp" #include "gc/g1/g1CardSet.hpp"
#include "memory/allocation.hpp" #include "memory/allocation.hpp"
#include "runtime/atomicAccess.hpp" #include "runtime/atomic.hpp"
#include "utilities/bitMap.hpp" #include "utilities/bitMap.hpp"
#include "utilities/globalDefinitions.hpp" #include "utilities/globalDefinitions.hpp"
@ -67,7 +67,7 @@ class G1CardSetInlinePtr : public StackObj {
using ContainerPtr = G1CardSet::ContainerPtr; using ContainerPtr = G1CardSet::ContainerPtr;
ContainerPtr volatile * _value_addr; Atomic<ContainerPtr>* _value_addr;
ContainerPtr _value; ContainerPtr _value;
static const uint SizeFieldLen = 3; static const uint SizeFieldLen = 3;
@ -103,7 +103,7 @@ public:
explicit G1CardSetInlinePtr(ContainerPtr value) : explicit G1CardSetInlinePtr(ContainerPtr value) :
G1CardSetInlinePtr(nullptr, value) {} G1CardSetInlinePtr(nullptr, value) {}
G1CardSetInlinePtr(ContainerPtr volatile* value_addr, ContainerPtr value) : _value_addr(value_addr), _value(value) { G1CardSetInlinePtr(Atomic<ContainerPtr>* value_addr, ContainerPtr value) : _value_addr(value_addr), _value(value) {
assert(G1CardSet::container_type(_value) == G1CardSet::ContainerInlinePtr, "Value " PTR_FORMAT " is not a valid G1CardSetInlinePtr.", p2i(_value)); assert(G1CardSet::container_type(_value) == G1CardSet::ContainerInlinePtr, "Value " PTR_FORMAT " is not a valid G1CardSetInlinePtr.", p2i(_value));
} }
@ -145,13 +145,13 @@ public:
// All but inline pointers are of this kind. For those, card entries are stored // All but inline pointers are of this kind. For those, card entries are stored
// directly in the ContainerPtr of the ConcurrentHashTable node. // directly in the ContainerPtr of the ConcurrentHashTable node.
class G1CardSetContainer { class G1CardSetContainer {
uintptr_t _ref_count; Atomic<uintptr_t> _ref_count;
protected: protected:
~G1CardSetContainer() = default; ~G1CardSetContainer() = default;
public: public:
G1CardSetContainer() : _ref_count(3) { } G1CardSetContainer() : _ref_count(3) { }
uintptr_t refcount() const { return AtomicAccess::load_acquire(&_ref_count); } uintptr_t refcount() const { return _ref_count.load_acquire(); }
bool try_increment_refcount(); bool try_increment_refcount();
@ -172,7 +172,7 @@ public:
using ContainerPtr = G1CardSet::ContainerPtr; using ContainerPtr = G1CardSet::ContainerPtr;
private: private:
EntryCountType _size; EntryCountType _size;
EntryCountType volatile _num_entries; Atomic<EntryCountType> _num_entries;
// VLA implementation. // VLA implementation.
EntryDataType _data[1]; EntryDataType _data[1];
@ -180,10 +180,10 @@ private:
static const EntryCountType EntryMask = LockBitMask - 1; static const EntryCountType EntryMask = LockBitMask - 1;
class G1CardSetArrayLocker : public StackObj { class G1CardSetArrayLocker : public StackObj {
EntryCountType volatile* _num_entries_addr; Atomic<EntryCountType>* _num_entries_addr;
EntryCountType _local_num_entries; EntryCountType _local_num_entries;
public: public:
G1CardSetArrayLocker(EntryCountType volatile* value); G1CardSetArrayLocker(Atomic<EntryCountType>* value);
EntryCountType num_entries() const { return _local_num_entries; } EntryCountType num_entries() const { return _local_num_entries; }
void inc_num_entries() { void inc_num_entries() {
@ -192,7 +192,7 @@ private:
} }
~G1CardSetArrayLocker() { ~G1CardSetArrayLocker() {
AtomicAccess::release_store(_num_entries_addr, _local_num_entries); _num_entries_addr->release_store(_local_num_entries);
} }
}; };
@ -213,7 +213,7 @@ public:
template <class CardVisitor> template <class CardVisitor>
void iterate(CardVisitor& found); void iterate(CardVisitor& found);
size_t num_entries() const { return _num_entries & EntryMask; } size_t num_entries() const { return _num_entries.load_relaxed() & EntryMask; }
static size_t header_size_in_bytes(); static size_t header_size_in_bytes();
@ -223,7 +223,7 @@ public:
}; };
class G1CardSetBitMap : public G1CardSetContainer { class G1CardSetBitMap : public G1CardSetContainer {
size_t _num_bits_set; Atomic<size_t> _num_bits_set;
BitMap::bm_word_t _bits[1]; BitMap::bm_word_t _bits[1];
public: public:
@ -236,7 +236,7 @@ public:
return bm.at(card_idx); return bm.at(card_idx);
} }
uint num_bits_set() const { return (uint)_num_bits_set; } uint num_bits_set() const { return (uint)_num_bits_set.load_relaxed(); }
template <class CardVisitor> template <class CardVisitor>
void iterate(CardVisitor& found, size_t const size_in_bits, uint offset); void iterate(CardVisitor& found, size_t const size_in_bits, uint offset);
@ -255,10 +255,10 @@ class G1CardSetHowl : public G1CardSetContainer {
public: public:
typedef uint EntryCountType; typedef uint EntryCountType;
using ContainerPtr = G1CardSet::ContainerPtr; using ContainerPtr = G1CardSet::ContainerPtr;
EntryCountType volatile _num_entries; Atomic<EntryCountType> _num_entries;
private: private:
// VLA implementation. // VLA implementation.
ContainerPtr _buckets[1]; Atomic<ContainerPtr> _buckets[1];
// Do not add class member variables beyond this point. // Do not add class member variables beyond this point.
// Iterates over the given ContainerPtr with at index in this Howl card set, // Iterates over the given ContainerPtr with at index in this Howl card set,
@ -268,14 +268,14 @@ private:
ContainerPtr at(EntryCountType index) const; ContainerPtr at(EntryCountType index) const;
ContainerPtr const* buckets() const; Atomic<ContainerPtr> const* buckets() const;
public: public:
G1CardSetHowl(EntryCountType card_in_region, G1CardSetConfiguration* config); G1CardSetHowl(EntryCountType card_in_region, G1CardSetConfiguration* config);
ContainerPtr const* container_addr(EntryCountType index) const; Atomic<ContainerPtr> const* container_addr(EntryCountType index) const;
ContainerPtr* container_addr(EntryCountType index); Atomic<ContainerPtr>* container_addr(EntryCountType index);
bool contains(uint card_idx, G1CardSetConfiguration* config); bool contains(uint card_idx, G1CardSetConfiguration* config);
// Iterates over all ContainerPtrs in this Howl card set, applying a CardOrRangeVisitor // Iterates over all ContainerPtrs in this Howl card set, applying a CardOrRangeVisitor

49
src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp
View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
* *
* This code is free software; you can redistribute it and/or modify it * This code is free software; you can redistribute it and/or modify it
@ -67,7 +67,7 @@ inline G1AddCardResult G1CardSetInlinePtr::add(uint card_idx, uint bits_per_card
return Overflow; return Overflow;
} }
ContainerPtr new_value = merge(_value, card_idx, num_cards, bits_per_card); ContainerPtr new_value = merge(_value, card_idx, num_cards, bits_per_card);
ContainerPtr old_value = AtomicAccess::cmpxchg(_value_addr, _value, new_value, memory_order_relaxed); ContainerPtr old_value = _value_addr->compare_exchange(_value, new_value, memory_order_relaxed);
if (_value == old_value) { if (_value == old_value) {
return Added; return Added;
} }
@ -126,7 +126,7 @@ inline bool G1CardSetContainer::try_increment_refcount() {
} }
uintptr_t new_value = old_value + 2; uintptr_t new_value = old_value + 2;
uintptr_t ref_count = AtomicAccess::cmpxchg(&_ref_count, old_value, new_value); uintptr_t ref_count = _ref_count.compare_exchange(old_value, new_value);
if (ref_count == old_value) { if (ref_count == old_value) {
return true; return true;
} }
@ -137,7 +137,7 @@ inline bool G1CardSetContainer::try_increment_refcount() {
inline uintptr_t G1CardSetContainer::decrement_refcount() { inline uintptr_t G1CardSetContainer::decrement_refcount() {
uintptr_t old_value = refcount(); uintptr_t old_value = refcount();
assert((old_value & 0x1) != 0 && old_value >= 3, "precondition"); assert((old_value & 0x1) != 0 && old_value >= 3, "precondition");
return AtomicAccess::sub(&_ref_count, 2u); return _ref_count.sub_then_fetch(2u);
} }
inline G1CardSetArray::G1CardSetArray(uint card_in_region, EntryCountType num_cards) : inline G1CardSetArray::G1CardSetArray(uint card_in_region, EntryCountType num_cards) :
@ -149,14 +149,13 @@ inline G1CardSetArray::G1CardSetArray(uint card_in_region, EntryCountType num_ca
*entry_addr(0) = checked_cast<EntryDataType>(card_in_region); *entry_addr(0) = checked_cast<EntryDataType>(card_in_region);
} }
inline G1CardSetArray::G1CardSetArrayLocker::G1CardSetArrayLocker(EntryCountType volatile* num_entries_addr) : inline G1CardSetArray::G1CardSetArrayLocker::G1CardSetArrayLocker(Atomic<EntryCountType>* num_entries_addr) :
_num_entries_addr(num_entries_addr) { _num_entries_addr(num_entries_addr) {
SpinYield s; SpinYield s;
EntryCountType num_entries = AtomicAccess::load(_num_entries_addr) & EntryMask; EntryCountType num_entries = _num_entries_addr->load_relaxed() & EntryMask;
while (true) { while (true) {
EntryCountType old_value = AtomicAccess::cmpxchg(_num_entries_addr, EntryCountType old_value = _num_entries_addr->compare_exchange(num_entries,
num_entries, (EntryCountType)(num_entries | LockBitMask));
(EntryCountType)(num_entries | LockBitMask));
if (old_value == num_entries) { if (old_value == num_entries) {
// Succeeded locking the array. // Succeeded locking the array.
_local_num_entries = num_entries; _local_num_entries = num_entries;
@ -174,7 +173,7 @@ inline G1CardSetArray::EntryDataType const* G1CardSetArray::base_addr() const {
} }
inline G1CardSetArray::EntryDataType const* G1CardSetArray::entry_addr(EntryCountType index) const { inline G1CardSetArray::EntryDataType const* G1CardSetArray::entry_addr(EntryCountType index) const {
assert(index < _num_entries, "precondition"); assert(index < _num_entries.load_relaxed(), "precondition");
return base_addr() + index; return base_addr() + index;
} }
@ -189,7 +188,7 @@ inline G1CardSetArray::EntryDataType G1CardSetArray::at(EntryCountType index) co
inline G1AddCardResult G1CardSetArray::add(uint card_idx) { inline G1AddCardResult G1CardSetArray::add(uint card_idx) {
assert(card_idx < (1u << (sizeof(EntryDataType) * BitsPerByte)), assert(card_idx < (1u << (sizeof(EntryDataType) * BitsPerByte)),
"Card index %u does not fit allowed card value range.", card_idx); "Card index %u does not fit allowed card value range.", card_idx);
EntryCountType num_entries = AtomicAccess::load_acquire(&_num_entries) & EntryMask; EntryCountType num_entries = _num_entries.load_acquire() & EntryMask;
EntryCountType idx = 0; EntryCountType idx = 0;
for (; idx < num_entries; idx++) { for (; idx < num_entries; idx++) {
if (at(idx) == card_idx) { if (at(idx) == card_idx) {
@ -223,7 +222,7 @@ inline G1AddCardResult G1CardSetArray::add(uint card_idx) {
} }
inline bool G1CardSetArray::contains(uint card_idx) { inline bool G1CardSetArray::contains(uint card_idx) {
EntryCountType num_entries = AtomicAccess::load_acquire(&_num_entries) & EntryMask; EntryCountType num_entries = _num_entries.load_acquire() & EntryMask;
for (EntryCountType idx = 0; idx < num_entries; idx++) { for (EntryCountType idx = 0; idx < num_entries; idx++) {
if (at(idx) == card_idx) { if (at(idx) == card_idx) {
@ -235,7 +234,7 @@ inline bool G1CardSetArray::contains(uint card_idx) {
template <class CardVisitor> template <class CardVisitor>
void G1CardSetArray::iterate(CardVisitor& found) { void G1CardSetArray::iterate(CardVisitor& found) {
EntryCountType num_entries = AtomicAccess::load_acquire(&_num_entries) & EntryMask; EntryCountType num_entries = _num_entries.load_acquire() & EntryMask;
for (EntryCountType idx = 0; idx < num_entries; idx++) { for (EntryCountType idx = 0; idx < num_entries; idx++) {
found(at(idx)); found(at(idx));
} }
@ -256,11 +255,11 @@ inline G1CardSetBitMap::G1CardSetBitMap(uint card_in_region, uint size_in_bits)
inline G1AddCardResult G1CardSetBitMap::add(uint card_idx, size_t threshold, size_t size_in_bits) { inline G1AddCardResult G1CardSetBitMap::add(uint card_idx, size_t threshold, size_t size_in_bits) {
BitMapView bm(_bits, size_in_bits); BitMapView bm(_bits, size_in_bits);
if (_num_bits_set >= threshold) { if (_num_bits_set.load_relaxed() >= threshold) {
return bm.at(card_idx) ? Found : Overflow; return bm.at(card_idx) ? Found : Overflow;
} }
if (bm.par_set_bit(card_idx)) { if (bm.par_set_bit(card_idx)) {
AtomicAccess::inc(&_num_bits_set, memory_order_relaxed); _num_bits_set.add_then_fetch(1u, memory_order_relaxed);
return Added; return Added;
} }
return Found; return Found;
@ -276,22 +275,22 @@ inline size_t G1CardSetBitMap::header_size_in_bytes() {
return offset_of(G1CardSetBitMap, _bits); return offset_of(G1CardSetBitMap, _bits);
} }
inline G1CardSetHowl::ContainerPtr const* G1CardSetHowl::container_addr(EntryCountType index) const { inline Atomic<G1CardSetHowl::ContainerPtr> const* G1CardSetHowl::container_addr(EntryCountType index) const {
assert(index < _num_entries, "precondition"); assert(index < _num_entries.load_relaxed(), "precondition");
return buckets() + index; return buckets() + index;
} }
inline G1CardSetHowl::ContainerPtr* G1CardSetHowl::container_addr(EntryCountType index) { inline Atomic<G1CardSetHowl::ContainerPtr>* G1CardSetHowl::container_addr(EntryCountType index) {
return const_cast<ContainerPtr*>(const_cast<const G1CardSetHowl*>(this)->container_addr(index)); return const_cast<Atomic<ContainerPtr>*>(const_cast<const G1CardSetHowl*>(this)->container_addr(index));
} }
inline G1CardSetHowl::ContainerPtr G1CardSetHowl::at(EntryCountType index) const { inline G1CardSetHowl::ContainerPtr G1CardSetHowl::at(EntryCountType index) const {
return *container_addr(index); return (*container_addr(index)).load_relaxed();
} }
inline G1CardSetHowl::ContainerPtr const* G1CardSetHowl::buckets() const { inline Atomic<G1CardSetHowl::ContainerPtr> const* G1CardSetHowl::buckets() const {
const void* ptr = reinterpret_cast<const char*>(this) + header_size_in_bytes(); const void* ptr = reinterpret_cast<const char*>(this) + header_size_in_bytes();
return reinterpret_cast<ContainerPtr const*>(ptr); return reinterpret_cast<Atomic<ContainerPtr> const*>(ptr);
} }
inline G1CardSetHowl::G1CardSetHowl(EntryCountType card_in_region, G1CardSetConfiguration* config) : inline G1CardSetHowl::G1CardSetHowl(EntryCountType card_in_region, G1CardSetConfiguration* config) :
@ -300,7 +299,7 @@ inline G1CardSetHowl::G1CardSetHowl(EntryCountType card_in_region, G1CardSetConf
EntryCountType num_buckets = config->num_buckets_in_howl(); EntryCountType num_buckets = config->num_buckets_in_howl();
EntryCountType bucket = config->howl_bucket_index(card_in_region); EntryCountType bucket = config->howl_bucket_index(card_in_region);
for (uint i = 0; i < num_buckets; ++i) { for (uint i = 0; i < num_buckets; ++i) {
*container_addr(i) = G1CardSetInlinePtr(); container_addr(i)->store_relaxed(G1CardSetInlinePtr());
if (i == bucket) { if (i == bucket) {
G1CardSetInlinePtr value(container_addr(i), at(i)); G1CardSetInlinePtr value(container_addr(i), at(i));
value.add(card_in_region, config->inline_ptr_bits_per_card(), config->max_cards_in_inline_ptr()); value.add(card_in_region, config->inline_ptr_bits_per_card(), config->max_cards_in_inline_ptr());
@ -310,8 +309,8 @@ inline G1CardSetHowl::G1CardSetHowl(EntryCountType card_in_region, G1CardSetConf
inline bool G1CardSetHowl::contains(uint card_idx, G1CardSetConfiguration* config) { inline bool G1CardSetHowl::contains(uint card_idx, G1CardSetConfiguration* config) {
EntryCountType bucket = config->howl_bucket_index(card_idx); EntryCountType bucket = config->howl_bucket_index(card_idx);
ContainerPtr* array_entry = container_addr(bucket); Atomic<ContainerPtr>* array_entry = container_addr(bucket);
ContainerPtr container = AtomicAccess::load_acquire(array_entry); ContainerPtr container = array_entry->load_acquire();
switch (G1CardSet::container_type(container)) { switch (G1CardSet::container_type(container)) {
case G1CardSet::ContainerArrayOfCards: { case G1CardSet::ContainerArrayOfCards: {

3
src/hotspot/share/gc/g1/g1CardSetMemory.cpp
View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
* *
* This code is free software; you can redistribute it and/or modify it * This code is free software; you can redistribute it and/or modify it
@ -26,7 +26,6 @@
#include "gc/g1/g1CardSetContainers.inline.hpp" #include "gc/g1/g1CardSetContainers.inline.hpp"
#include "gc/g1/g1CardSetMemory.inline.hpp" #include "gc/g1/g1CardSetMemory.inline.hpp"
#include "gc/g1/g1MonotonicArena.inline.hpp" #include "gc/g1/g1MonotonicArena.inline.hpp"
#include "runtime/atomicAccess.hpp"
#include "utilities/ostream.hpp" #include "utilities/ostream.hpp"
G1CardSetAllocator::G1CardSetAllocator(const char* name, G1CardSetAllocator::G1CardSetAllocator(const char* name,

19
test/hotspot/gtest/gc/g1/test_g1CardSetContainers.cpp
View File

@ -25,6 +25,7 @@
#include "gc/g1/g1HeapRegionBounds.inline.hpp" #include "gc/g1/g1HeapRegionBounds.inline.hpp"
#include "gc/shared/cardTable.hpp" #include "gc/shared/cardTable.hpp"
#include "memory/allocation.inline.hpp" #include "memory/allocation.inline.hpp"
#include "runtime/atomic.hpp"
#include "utilities/globalDefinitions.hpp" #include "utilities/globalDefinitions.hpp"
#include "utilities/powerOfTwo.hpp" #include "utilities/powerOfTwo.hpp"
#include "unittest.hpp" #include "unittest.hpp"
@ -82,48 +83,48 @@ void G1CardSetContainersTest::cardset_inlineptr_test(uint bits_per_card) {
G1AddCardResult res; G1AddCardResult res;
G1CardSet::ContainerPtr value = G1CardSetInlinePtr(); Atomic<G1CardSet::ContainerPtr> value{};
for (uint i = 0; i < CardsPerSet; i++) { for (uint i = 0; i < CardsPerSet; i++) {
{ {
G1CardSetInlinePtr cards(&value, value); G1CardSetInlinePtr cards(&value, value.load_relaxed());
res = cards.add(i + 1, bits_per_card, CardsPerSet); res = cards.add(i + 1, bits_per_card, CardsPerSet);
ASSERT_TRUE(res == Added); ASSERT_TRUE(res == Added);
} }
{ {
G1CardSetInlinePtr cards(&value, value); G1CardSetInlinePtr cards(&value, value.load_relaxed());
ASSERT_TRUE(cards.contains(i + 1, bits_per_card)); ASSERT_TRUE(cards.contains(i + 1, bits_per_card));
} }
} }
for (uint i = 0; i < CardsPerSet; i++) { for (uint i = 0; i < CardsPerSet; i++) {
G1CardSetInlinePtr cards(value); G1CardSetInlinePtr cards(value.load_relaxed());
ASSERT_TRUE(cards.contains(i + 1, bits_per_card)); ASSERT_TRUE(cards.contains(i + 1, bits_per_card));
} }
// Try to add again, should all return that the card had been added. // Try to add again, should all return that the card had been added.
for (uint i = 0; i < CardsPerSet; i++) { for (uint i = 0; i < CardsPerSet; i++) {
G1CardSetInlinePtr cards(&value, value); G1CardSetInlinePtr cards(&value, value.load_relaxed());
res = cards.add(i + 1, bits_per_card, CardsPerSet); res = cards.add(i + 1, bits_per_card, CardsPerSet);
ASSERT_TRUE(res == Found); ASSERT_TRUE(res == Found);
} }
// Should be no more space in set. // Should be no more space in set.
{ {
G1CardSetInlinePtr cards(&value, value); G1CardSetInlinePtr cards(&value, value.load_relaxed());
res = cards.add(CardsPerSet + 1, bits_per_card, CardsPerSet); res = cards.add(CardsPerSet + 1, bits_per_card, CardsPerSet);
ASSERT_TRUE(res == Overflow); ASSERT_TRUE(res == Overflow);
} }
// Cards should still be in the set. // Cards should still be in the set.
for (uint i = 0; i < CardsPerSet; i++) { for (uint i = 0; i < CardsPerSet; i++) {
G1CardSetInlinePtr cards(value); G1CardSetInlinePtr cards(value.load_relaxed());
ASSERT_TRUE(cards.contains(i + 1, bits_per_card)); ASSERT_TRUE(cards.contains(i + 1, bits_per_card));
} }
// Boundary cards should not be in the set. // Boundary cards should not be in the set.
{ {
G1CardSetInlinePtr cards(value); G1CardSetInlinePtr cards(value.load_relaxed());
ASSERT_TRUE(!cards.contains(0, bits_per_card)); ASSERT_TRUE(!cards.contains(0, bits_per_card));
ASSERT_TRUE(!cards.contains(CardsPerSet + 1, bits_per_card)); ASSERT_TRUE(!cards.contains(CardsPerSet + 1, bits_per_card));
} }
@ -131,7 +132,7 @@ void G1CardSetContainersTest::cardset_inlineptr_test(uint bits_per_card) {
// Verify iteration finds all cards too and only those. // Verify iteration finds all cards too and only those.
{ {
G1FindCardsInRange found(1, CardsPerSet); G1FindCardsInRange found(1, CardsPerSet);
G1CardSetInlinePtr cards(value); G1CardSetInlinePtr cards(value.load_relaxed());
cards.iterate(found, bits_per_card); cards.iterate(found, bits_per_card);
found.verify_all_found(); found.verify_all_found();
} }