diff --git a/src/hotspot/share/gc/z/zIndexDistributor.hpp b/src/hotspot/share/gc/z/zIndexDistributor.hpp index 94f146176d6..d4b9edcb5b4 100644 --- a/src/hotspot/share/gc/z/zIndexDistributor.hpp +++ b/src/hotspot/share/gc/z/zIndexDistributor.hpp @@ -24,6 +24,8 @@ #ifndef SHARE_GC_Z_ZINDEXDISTRIBUTOR_HPP #define SHARE_GC_Z_ZINDEXDISTRIBUTOR_HPP +#include + class ZIndexDistributor { private: void* _strategy; @@ -39,6 +41,10 @@ public: template void do_indices(Function function); + + // Returns a count that is max_count or larger and upholds the requirements + // for using the ZIndexDistributor strategy specfied by ZIndexDistributorStrategy + static size_t get_count(size_t max_count); }; #endif // SHARE_GC_Z_ZINDEXDISTRIBUTOR_HPP diff --git a/src/hotspot/share/gc/z/zIndexDistributor.inline.hpp b/src/hotspot/share/gc/z/zIndexDistributor.inline.hpp index c582be8fcee..163dd2ad685 100644 --- a/src/hotspot/share/gc/z/zIndexDistributor.inline.hpp +++ b/src/hotspot/share/gc/z/zIndexDistributor.inline.hpp @@ -32,11 +32,13 @@ #include "runtime/os.hpp" #include "runtime/thread.hpp" #include "utilities/align.hpp" +#include "utilities/powerOfTwo.hpp" class ZIndexDistributorStriped : public CHeapObj { static const int MemSize = 4096; + static const int StripeCount = MemSize / ZCacheLineSize; - const int _max_index; + const int _count; // For claiming a stripe volatile int _claim_stripe; // For claiming inside a stripe @@ -51,8 +53,8 @@ class ZIndexDistributorStriped : public CHeapObj { } public: - ZIndexDistributorStriped(int max_index) - : _max_index(max_index), + ZIndexDistributorStriped(int count) + : _count(count), _claim_stripe(0), _mem() { memset(_mem, 0, MemSize + ZCacheLineSize); @@ -60,11 +62,10 @@ public: template void do_indices(Function function) { - const int count = MemSize / ZCacheLineSize; - const int stripe_max = _max_index / count; + const int stripe_max = _count / StripeCount; // Use claiming - for (int i; (i = claim_stripe()) < count;) { + for (int i; (i = claim_stripe()) < StripeCount;) { for (int index; (index = Atomic::fetch_then_add(claim_addr(i), 1, memory_order_relaxed)) < stripe_max;) { if (!function(i * stripe_max + index)) { return; @@ -73,7 +74,7 @@ public: } // Use stealing - for (int i = 0; i < count; i++) { + for (int i = 0; i < StripeCount; i++) { for (int index; (index = Atomic::fetch_then_add(claim_addr(i), 1, memory_order_relaxed)) < stripe_max;) { if (!function(i * stripe_max + index)) { return; @@ -81,6 +82,11 @@ public: } } } + + static size_t get_count(size_t max_count) { + // Must be multiple of the StripeCount + return align_up(max_count, StripeCount); + } }; class ZIndexDistributorClaimTree : public CHeapObj { @@ -290,6 +296,12 @@ public: claim_and_do(function, indices, 0 /* level */); steal_and_do(function, indices, 0 /* level */); } + + static size_t get_count(size_t max_count) { + // Must be at least claim_level_size(ClaimLevels) and a power of two + const size_t min_count = claim_level_size(ClaimLevels); + return round_up_power_of_2(MAX2(max_count, min_count)); + } }; // Using dynamically allocated objects just to be able to evaluate @@ -328,4 +340,17 @@ inline void ZIndexDistributor::do_indices(Function function) { }; } +inline size_t ZIndexDistributor::get_count(size_t max_count) { + size_t required_count; + switch (ZIndexDistributorStrategy) { + case 0: required_count = ZIndexDistributorClaimTree::get_count(max_count); break; + case 1: required_count = ZIndexDistributorStriped::get_count(max_count); break; + default: fatal("Unknown ZIndexDistributorStrategy"); + }; + + assert(max_count <= required_count, "unsupported max_count: %zu", max_count); + + return required_count; +} + #endif // SHARE_GC_Z_ZINDEXDISTRIBUTOR_INLINE_HPP diff --git a/src/hotspot/share/gc/z/zMemory.cpp b/src/hotspot/share/gc/z/zMemory.cpp index 14578c2db8a..8ccde9f45a2 100644 --- a/src/hotspot/share/gc/z/zMemory.cpp +++ b/src/hotspot/share/gc/z/zMemory.cpp @@ -100,6 +100,18 @@ zoffset ZMemoryManager::peek_low_address() const { return zoffset(UINTPTR_MAX); } +zoffset_end ZMemoryManager::peak_high_address_end() const { + ZLocker locker(&_lock); + + const ZMemory* const area = _freelist.last(); + if (area != nullptr) { + return area->end(); + } + + // Out of memory + return zoffset_end(UINTPTR_MAX); +} + zoffset ZMemoryManager::alloc_low_address(size_t size) { ZLocker locker(&_lock); diff --git a/src/hotspot/share/gc/z/zMemory.hpp b/src/hotspot/share/gc/z/zMemory.hpp index ce9100f8026..229d5a7c50c 100644 --- a/src/hotspot/share/gc/z/zMemory.hpp +++ b/src/hotspot/share/gc/z/zMemory.hpp @@ -86,6 +86,7 @@ public: void register_callbacks(const Callbacks& callbacks); zoffset peek_low_address() const; + zoffset_end peak_high_address_end() const; zoffset alloc_low_address(size_t size); zoffset alloc_low_address_at_most(size_t size, size_t* allocated); zoffset alloc_high_address(size_t size); diff --git a/src/hotspot/share/gc/z/zPageTable.cpp b/src/hotspot/share/gc/z/zPageTable.cpp index 9a4bbc85d04..d960270c451 100644 --- a/src/hotspot/share/gc/z/zPageTable.cpp +++ b/src/hotspot/share/gc/z/zPageTable.cpp @@ -23,13 +23,22 @@ #include "gc/z/zAddress.hpp" #include "gc/z/zGranuleMap.inline.hpp" +#include "gc/z/zIndexDistributor.inline.hpp" #include "gc/z/zPage.inline.hpp" #include "gc/z/zPageTable.inline.hpp" #include "runtime/orderAccess.hpp" #include "utilities/debug.hpp" +static size_t get_max_offset_for_map() { + // The page table has (ZAddressOffsetMax >> ZGranuleSizeShift) slots + const size_t max_count = ZAddressOffsetMax >> ZGranuleSizeShift; + const size_t required_count = ZIndexDistributor::get_count(max_count); + + return required_count << ZGranuleSizeShift; +} + ZPageTable::ZPageTable() - : _map(ZAddressOffsetMax) {} + : _map(get_max_offset_for_map()) {} void ZPageTable::insert(ZPage* page) { const zoffset offset = page->start(); diff --git a/src/hotspot/share/gc/z/zPageTable.hpp b/src/hotspot/share/gc/z/zPageTable.hpp index e809a0ac0ca..fadf550d996 100644 --- a/src/hotspot/share/gc/z/zPageTable.hpp +++ b/src/hotspot/share/gc/z/zPageTable.hpp @@ -45,6 +45,8 @@ private: public: ZPageTable(); + int count() const; + ZPage* get(zaddress addr) const; ZPage* get(volatile zpointer* p) const; diff --git a/src/hotspot/share/gc/z/zPageTable.inline.hpp b/src/hotspot/share/gc/z/zPageTable.inline.hpp index 79a2d297df8..583017d5c9c 100644 --- a/src/hotspot/share/gc/z/zPageTable.inline.hpp +++ b/src/hotspot/share/gc/z/zPageTable.inline.hpp @@ -32,6 +32,15 @@ #include "gc/z/zPage.inline.hpp" #include "gc/z/zPageAllocator.inline.hpp" +#include + +inline int ZPageTable::count() const { + const size_t size = _map._size; + assert(size <= std::numeric_limits::max(), "Invalid page table size"); + + return static_cast(size); +} + inline ZPage* ZPageTable::get(zaddress addr) const { assert(!is_null(addr), "Invalid address"); return _map.get(ZAddress::offset(addr)); @@ -64,7 +73,7 @@ inline bool ZPageTableIterator::next(ZPage** page) { inline ZPageTableParallelIterator::ZPageTableParallelIterator(const ZPageTable* table) : _table(table), - _index_distributor(int(ZAddressOffsetMax >> ZGranuleSizeShift)) {} + _index_distributor(table->count()) {} template inline void ZPageTableParallelIterator::do_pages(Function function) { diff --git a/src/hotspot/share/gc/z/zVirtualMemory.cpp b/src/hotspot/share/gc/z/zVirtualMemory.cpp index ccc30626a3f..21152cf4db6 100644 --- a/src/hotspot/share/gc/z/zVirtualMemory.cpp +++ b/src/hotspot/share/gc/z/zVirtualMemory.cpp @@ -48,6 +48,9 @@ ZVirtualMemoryManager::ZVirtualMemoryManager(size_t max_capacity) return; } + // Set ZAddressOffsetMax to the highest address end available after reservation + ZAddressOffsetMax = untype(highest_available_address_end()); + // Initialize platform specific parts after reserving address space pd_initialize_after_reserve(); diff --git a/src/hotspot/share/gc/z/zVirtualMemory.hpp b/src/hotspot/share/gc/z/zVirtualMemory.hpp index a2df86262ea..57a377ab61b 100644 --- a/src/hotspot/share/gc/z/zVirtualMemory.hpp +++ b/src/hotspot/share/gc/z/zVirtualMemory.hpp @@ -77,6 +77,7 @@ public: size_t reserved() const; zoffset lowest_available_address() const; + zoffset_end highest_available_address_end() const; ZVirtualMemory alloc(size_t size, bool force_low_address); void free(const ZVirtualMemory& vmem); diff --git a/src/hotspot/share/gc/z/zVirtualMemory.inline.hpp b/src/hotspot/share/gc/z/zVirtualMemory.inline.hpp index 972e51a143b..9d5fe7dd57a 100644 --- a/src/hotspot/share/gc/z/zVirtualMemory.inline.hpp +++ b/src/hotspot/share/gc/z/zVirtualMemory.inline.hpp @@ -65,4 +65,8 @@ inline zoffset ZVirtualMemoryManager::lowest_available_address() const { return _manager.peek_low_address(); } +inline zoffset_end ZVirtualMemoryManager::highest_available_address_end() const { + return _manager.peak_high_address_end(); +} + #endif // SHARE_GC_Z_ZVIRTUALMEMORY_INLINE_HPP