8350851: ZGC: Reduce size of ZAddressOffsetMax scaling data structures

Reviewed-by: eosterlund, jsikstro

src/hotspot/share/gc/z/zIndexDistributor.hpp

@@ -24,6 +24,8 @@
 #ifndef SHARE_GC_Z_ZINDEXDISTRIBUTOR_HPP
 #define SHARE_GC_Z_ZINDEXDISTRIBUTOR_HPP
 
+#include <utilities/globalDefinitions.hpp>
+
 class ZIndexDistributor {
 private:
   void* _strategy;
@@ -39,6 +41,10 @@ public:
 
   template <typename Function>
   void do_indices(Function function);
+
+  // Returns a count that is max_count or larger and upholds the requirements
+  // for using the ZIndexDistributor strategy specfied by ZIndexDistributorStrategy
+  static size_t get_count(size_t max_count);
 };
 
 #endif // SHARE_GC_Z_ZINDEXDISTRIBUTOR_HPP

src/hotspot/share/gc/z/zIndexDistributor.inline.hpp

@@ -32,11 +32,13 @@
 #include "runtime/os.hpp"
 #include "runtime/thread.hpp"
 #include "utilities/align.hpp"
+#include "utilities/powerOfTwo.hpp"
 
 class ZIndexDistributorStriped : public CHeapObj<mtGC> {
   static const int MemSize = 4096;
+  static const int StripeCount = MemSize / ZCacheLineSize;
 
-  const int _max_index;
+  const int _count;
   // For claiming a stripe
   volatile int _claim_stripe;
   // For claiming inside a stripe
@@ -51,8 +53,8 @@ class ZIndexDistributorStriped : public CHeapObj<mtGC> {
   }
 
 public:
-  ZIndexDistributorStriped(int max_index)
-    : _max_index(max_index),
+  ZIndexDistributorStriped(int count)
+    : _count(count),
       _claim_stripe(0),
       _mem() {
     memset(_mem, 0, MemSize + ZCacheLineSize);
@@ -60,11 +62,10 @@ public:
 
   template <typename Function>
   void do_indices(Function function) {
-    const int count = MemSize / ZCacheLineSize;
-    const int stripe_max = _max_index / count;
+    const int stripe_max = _count / StripeCount;
 
     // Use claiming
-    for (int i; (i = claim_stripe()) < count;) {
+    for (int i; (i = claim_stripe()) < StripeCount;) {
       for (int index; (index = Atomic::fetch_then_add(claim_addr(i), 1, memory_order_relaxed)) < stripe_max;) {
         if (!function(i * stripe_max + index)) {
           return;
@@ -73,7 +74,7 @@ public:
     }
 
     // Use stealing
-    for (int i = 0; i < count; i++) {
+    for (int i = 0; i < StripeCount; i++) {
       for (int index; (index = Atomic::fetch_then_add(claim_addr(i), 1, memory_order_relaxed)) < stripe_max;) {
         if (!function(i * stripe_max + index)) {
           return;
@@ -81,6 +82,11 @@ public:
       }
     }
   }
+
+  static size_t get_count(size_t max_count) {
+    // Must be multiple of the StripeCount
+    return align_up(max_count, StripeCount);
+  }
 };
 
 class ZIndexDistributorClaimTree : public CHeapObj<mtGC> {
@@ -290,6 +296,12 @@ public:
     claim_and_do(function, indices, 0 /* level */);
     steal_and_do(function, indices, 0 /* level */);
   }
+
+  static size_t get_count(size_t max_count) {
+    // Must be at least claim_level_size(ClaimLevels) and a power of two
+    const size_t min_count = claim_level_size(ClaimLevels);
+    return round_up_power_of_2(MAX2(max_count, min_count));
+  }
 };
 
 // Using dynamically allocated objects just to be able to evaluate
@@ -328,4 +340,17 @@ inline void ZIndexDistributor::do_indices(Function function) {
   };
 }
 
+inline size_t ZIndexDistributor::get_count(size_t max_count) {
+  size_t required_count;
+  switch (ZIndexDistributorStrategy) {
+  case 0: required_count = ZIndexDistributorClaimTree::get_count(max_count); break;
+  case 1: required_count = ZIndexDistributorStriped::get_count(max_count); break;
+  default: fatal("Unknown ZIndexDistributorStrategy");
+  };
+
+  assert(max_count <= required_count, "unsupported max_count: %zu", max_count);
+
+  return required_count;
+}
+
 #endif // SHARE_GC_Z_ZINDEXDISTRIBUTOR_INLINE_HPP

src/hotspot/share/gc/z/zMemory.cpp

@@ -100,6 +100,18 @@ zoffset ZMemoryManager::peek_low_address() const {
   return zoffset(UINTPTR_MAX);
 }
 
+zoffset_end ZMemoryManager::peak_high_address_end() const {
+  ZLocker<ZLock> locker(&_lock);
+
+  const ZMemory* const area = _freelist.last();
+  if (area != nullptr) {
+    return area->end();
+  }
+
+  // Out of memory
+  return zoffset_end(UINTPTR_MAX);
+}
+
 zoffset ZMemoryManager::alloc_low_address(size_t size) {
   ZLocker<ZLock> locker(&_lock);
 

src/hotspot/share/gc/z/zMemory.hpp

@@ -86,6 +86,7 @@ public:
   void register_callbacks(const Callbacks& callbacks);
 
   zoffset peek_low_address() const;
+  zoffset_end peak_high_address_end() const;
   zoffset alloc_low_address(size_t size);
   zoffset alloc_low_address_at_most(size_t size, size_t* allocated);
   zoffset alloc_high_address(size_t size);

src/hotspot/share/gc/z/zPageTable.cpp

@@ -23,13 +23,22 @@
 
 #include "gc/z/zAddress.hpp"
 #include "gc/z/zGranuleMap.inline.hpp"
+#include "gc/z/zIndexDistributor.inline.hpp"
 #include "gc/z/zPage.inline.hpp"
 #include "gc/z/zPageTable.inline.hpp"
 #include "runtime/orderAccess.hpp"
 #include "utilities/debug.hpp"
 
+static size_t get_max_offset_for_map() {
+  // The page table has (ZAddressOffsetMax >> ZGranuleSizeShift) slots
+  const size_t max_count = ZAddressOffsetMax >> ZGranuleSizeShift;
+  const size_t required_count = ZIndexDistributor::get_count(max_count);
+
+  return required_count << ZGranuleSizeShift;
+}
+
 ZPageTable::ZPageTable()
-  : _map(ZAddressOffsetMax) {}
+  : _map(get_max_offset_for_map()) {}
 
 void ZPageTable::insert(ZPage* page) {
   const zoffset offset = page->start();

src/hotspot/share/gc/z/zPageTable.hpp

@@ -45,6 +45,8 @@ private:
 public:
   ZPageTable();
 
+  int count() const;
+
   ZPage* get(zaddress addr) const;
   ZPage* get(volatile zpointer* p) const;
 

src/hotspot/share/gc/z/zPageTable.inline.hpp

@@ -32,6 +32,15 @@
 #include "gc/z/zPage.inline.hpp"
 #include "gc/z/zPageAllocator.inline.hpp"
 
+#include <limits>
+
+inline int ZPageTable::count() const {
+  const size_t size = _map._size;
+  assert(size <= std::numeric_limits<int>::max(), "Invalid page table size");
+
+  return static_cast<int>(size);
+}
+
 inline ZPage* ZPageTable::get(zaddress addr) const {
   assert(!is_null(addr), "Invalid address");
   return _map.get(ZAddress::offset(addr));
@@ -64,7 +73,7 @@ inline bool ZPageTableIterator::next(ZPage** page) {
 
 inline ZPageTableParallelIterator::ZPageTableParallelIterator(const ZPageTable* table)
   : _table(table),
-    _index_distributor(int(ZAddressOffsetMax >> ZGranuleSizeShift)) {}
+    _index_distributor(table->count()) {}
 
 template <typename Function>
 inline void ZPageTableParallelIterator::do_pages(Function function) {

src/hotspot/share/gc/z/zVirtualMemory.cpp

@@ -48,6 +48,9 @@ ZVirtualMemoryManager::ZVirtualMemoryManager(size_t max_capacity)
     return;
   }
 
+  // Set ZAddressOffsetMax to the highest address end available after reservation
+  ZAddressOffsetMax = untype(highest_available_address_end());
+
   // Initialize platform specific parts after reserving address space
   pd_initialize_after_reserve();
 

src/hotspot/share/gc/z/zVirtualMemory.hpp

@@ -77,6 +77,7 @@ public:
 
   size_t reserved() const;
   zoffset lowest_available_address() const;
+  zoffset_end highest_available_address_end() const;
 
   ZVirtualMemory alloc(size_t size, bool force_low_address);
   void free(const ZVirtualMemory& vmem);

src/hotspot/share/gc/z/zVirtualMemory.inline.hpp

@@ -65,4 +65,8 @@ inline zoffset ZVirtualMemoryManager::lowest_available_address() const {
   return _manager.peek_low_address();
 }
 
+inline zoffset_end ZVirtualMemoryManager::highest_available_address_end() const {
+  return _manager.peak_high_address_end();
+}
+
 #endif // SHARE_GC_Z_ZVIRTUALMEMORY_INLINE_HPP