8255234: ZGC: Bulk allocate forwarding data structures

Co-authored-by: Albert Mingkun Yang <ayang@openjdk.org>
Co-authored-by: Per Liden <pliden@openjdk.org>
Reviewed-by: ayang, stefank

src/hotspot/share/gc/z/zAttachedArray.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,8 +32,12 @@ private:
   const size_t _length;
 
   static size_t object_size();
+  static size_t array_size(size_t length);
 
 public:
+  template <typename Allocator>
+  static void* alloc(Allocator* allocator, size_t length);
+
   static void* alloc(size_t length);
   static void free(ObjectT* obj);
 

src/hotspot/share/gc/z/zAttachedArray.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,13 +34,35 @@ inline size_t ZAttachedArray<ObjectT, ArrayT>::object_size() {
 }
 
 template <typename ObjectT, typename ArrayT>
-inline void* ZAttachedArray<ObjectT, ArrayT>::alloc(size_t length) {
-  const size_t array_size = sizeof(ArrayT) * length;
-  char* const addr = AllocateHeap(object_size() + array_size, mtGC);
-  ::new (addr + object_size()) ArrayT[length];
+inline size_t ZAttachedArray<ObjectT, ArrayT>::array_size(size_t length) {
+  return sizeof(ArrayT) * length;
+}
+
+template <typename ObjectT, typename ArrayT>
+template <typename Allocator>
+inline void* ZAttachedArray<ObjectT, ArrayT>::alloc(Allocator* allocator, size_t length) {
+  // Allocate memory for object and array
+  const size_t size = object_size() + array_size(length);
+  void* const addr = allocator->alloc(size);
+
+  // Placement new array
+  void* const array_addr = reinterpret_cast<char*>(addr) + object_size();
+  ::new (array_addr) ArrayT[length];
+
+  // Return pointer to object
   return addr;
 }
 
+template <typename ObjectT, typename ArrayT>
+inline void* ZAttachedArray<ObjectT, ArrayT>::alloc(size_t length) {
+  struct Allocator {
+    void* alloc(size_t size) const {
+      return AllocateHeap(size, mtGC);
+    }
+  } allocator;
+  return alloc(&allocator, length);
+}
+
 template <typename ObjectT, typename ArrayT>
 inline void ZAttachedArray<ObjectT, ArrayT>::free(ObjectT* obj) {
   FreeHeap(obj);

src/hotspot/share/gc/z/zForwarding.cpp

@@ -23,31 +23,7 @@
 
 #include "precompiled.hpp"
 #include "gc/z/zForwarding.inline.hpp"
-#include "gc/z/zPage.inline.hpp"
 #include "utilities/debug.hpp"
-#include "utilities/powerOfTwo.hpp"
-
-ZForwarding* ZForwarding::create(ZPage* page) {
-  // Allocate table for linear probing. The size of the table must be
-  // a power of two to allow for quick and inexpensive indexing/masking.
-  // The table is sized to have a load factor of 50%, i.e. sized to have
-  // double the number of entries actually inserted.
-  assert(page->live_objects() > 0, "Invalid value");
-  const size_t nentries = round_up_power_of_2(page->live_objects() * 2);
-  return ::new (AttachedArray::alloc(nentries)) ZForwarding(page, nentries);
-}
-
-void ZForwarding::destroy(ZForwarding* forwarding) {
-  AttachedArray::free(forwarding);
-}
-
-ZForwarding::ZForwarding(ZPage* page, size_t nentries) :
-    _virtual(page->virtual_memory()),
-    _object_alignment_shift(page->object_alignment_shift()),
-    _entries(nentries),
-    _page(page),
-    _refcount(1),
-    _pinned(false) {}
 
 void ZForwarding::verify() const {
   guarantee(_refcount > 0, "Invalid refcount");

src/hotspot/share/gc/z/zForwarding.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,7 @@
 #include "gc/z/zForwardingEntry.hpp"
 #include "gc/z/zVirtualMemory.hpp"
 
+class ZForwardingAllocator;
 class ZPage;
 
 typedef size_t ZForwardingCursor;
@@ -57,8 +58,8 @@ private:
   ZForwarding(ZPage* page, size_t nentries);
 
 public:
-  static ZForwarding* create(ZPage* page);
-  static void destroy(ZForwarding* forwarding);
+  static uint32_t nentries(const ZPage* page);
+  static ZForwarding* alloc(ZForwardingAllocator* allocator, ZPage* page);
 
   uintptr_t start() const;
   size_t size() const;

src/hotspot/share/gc/z/zForwarding.inline.hpp

@@ -26,11 +26,38 @@
 
 #include "gc/z/zAttachedArray.inline.hpp"
 #include "gc/z/zForwarding.hpp"
+#include "gc/z/zForwardingAllocator.inline.hpp"
 #include "gc/z/zHash.inline.hpp"
 #include "gc/z/zHeap.hpp"
+#include "gc/z/zPage.inline.hpp"
 #include "gc/z/zVirtualMemory.inline.hpp"
 #include "runtime/atomic.hpp"
 #include "utilities/debug.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+inline uint32_t ZForwarding::nentries(const ZPage* page) {
+  // The number returned by the function is used to size the hash table of
+  // forwarding entries for this page. This hash table uses linear probing.
+  // The size of the table must be a power of two to allow for quick and
+  // inexpensive indexing/masking. The table is also sized to have a load
+  // factor of 50%, i.e. sized to have double the number of entries actually
+  // inserted, to allow for good lookup/insert performance.
+  return round_up_power_of_2(page->live_objects() * 2);
+}
+
+inline ZForwarding* ZForwarding::alloc(ZForwardingAllocator* allocator, ZPage* page) {
+  const size_t nentries = ZForwarding::nentries(page);
+  void* const addr = AttachedArray::alloc(allocator, nentries);
+  return ::new (addr) ZForwarding(page, nentries);
+}
+
+inline ZForwarding::ZForwarding(ZPage* page, size_t nentries) :
+    _virtual(page->virtual_memory()),
+    _object_alignment_shift(page->object_alignment_shift()),
+    _entries(nentries),
+    _page(page),
+    _refcount(1),
+    _pinned(false) {}
 
 inline uintptr_t ZForwarding::start() const {
   return _virtual.start();

src/hotspot/share/gc/z/zForwardingAllocator.cpp

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zForwardingAllocator.hpp"
+#include "memory/allocation.inline.hpp"
+
+ZForwardingAllocator::ZForwardingAllocator() :
+    _start(NULL),
+    _end(NULL),
+    _top(NULL) {}
+
+ZForwardingAllocator::~ZForwardingAllocator() {
+  FREE_C_HEAP_ARRAY(char, _start);
+}
+
+void ZForwardingAllocator::reset(size_t size) {
+  _start = _top = REALLOC_C_HEAP_ARRAY(char, _start, size, mtGC);
+  _end = _start + size;
+}

src/hotspot/share/gc/z/zForwardingAllocator.hpp

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZFORWARDINGALLOCATOR_HPP
+#define SHARE_GC_Z_ZFORWARDINGALLOCATOR_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+class ZForwardingAllocator {
+private:
+  char* _start;
+  char* _end;
+  char* _top;
+
+public:
+  ZForwardingAllocator();
+  ~ZForwardingAllocator();
+
+  void reset(size_t size);
+  size_t size() const;
+  bool is_full() const;
+
+  void* alloc(size_t size);
+};
+
+#endif // SHARE_GC_Z_ZFORWARDINGALLOCATOR_HPP

src/hotspot/share/gc/z/zForwardingAllocator.inline.hpp

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZFORWARDINGALLOCATOR_INLINE_HPP
+#define SHARE_GC_Z_ZFORWARDINGALLOCATOR_INLINE_HPP
+
+#include "gc/z/zForwardingAllocator.hpp"
+#include "utilities/debug.hpp"
+
+inline size_t ZForwardingAllocator::size() const {
+  return _end - _start;
+}
+
+inline bool ZForwardingAllocator::is_full() const {
+  return _top == _end;
+}
+
+inline void* ZForwardingAllocator::alloc(size_t size) {
+  char* const addr = _top;
+  _top += size;
+  assert(_top <= _end, "Allocation should never fail");
+  return addr;
+}
+
+#endif // SHARE_GC_Z_ZFORWARDINGALLOCATOR_INLINE_HPP

src/hotspot/share/gc/z/zRelocationSet.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,35 +22,53 @@
  */
 
 #include "precompiled.hpp"
-#include "gc/z/zForwarding.hpp"
+#include "gc/z/zForwarding.inline.hpp"
+#include "gc/z/zForwardingAllocator.inline.hpp"
 #include "gc/z/zRelocationSet.hpp"
+#include "gc/z/zStat.hpp"
 #include "memory/allocation.hpp"
+#include "utilities/debug.hpp"
 
 ZRelocationSet::ZRelocationSet() :
+    _allocator(),
     _forwardings(NULL),
     _nforwardings(0) {}
 
-void ZRelocationSet::populate(ZPage* const* group0, size_t ngroup0,
-                              ZPage* const* group1, size_t ngroup1) {
-  _nforwardings = ngroup0 + ngroup1;
-  _forwardings = REALLOC_C_HEAP_ARRAY(ZForwarding*, _forwardings, _nforwardings, mtGC);
+void ZRelocationSet::populate(ZPage* const* small, size_t nsmall,
+                              ZPage* const* medium, size_t nmedium,
+                              size_t forwarding_entries) {
+  // Set relocation set length
+  _nforwardings = nsmall + nmedium;
 
+  // Initialize forwarding allocator to have room for the
+  // relocation set, all forwardings, and all forwarding entries.
+  const size_t relocation_set_size = _nforwardings * sizeof(ZForwarding*);
+  const size_t forwardings_size = _nforwardings * sizeof(ZForwarding);
+  const size_t forwarding_entries_size = forwarding_entries * sizeof(ZForwardingEntry);
+  _allocator.reset(relocation_set_size + forwardings_size + forwarding_entries_size);
+
+  // Allocate relocation set
+  _forwardings = new (_allocator.alloc(relocation_set_size)) ZForwarding*[_nforwardings];
+
+  // Populate relocation set array
   size_t j = 0;
 
-  // Populate group 0
-  for (size_t i = 0; i < ngroup0; i++) {
-    _forwardings[j++] = ZForwarding::create(group0[i]);
+  // Populate medium pages
+  for (size_t i = 0; i < nmedium; i++) {
+    _forwardings[j++] = ZForwarding::alloc(&_allocator, medium[i]);
   }
 
-  // Populate group 1
-  for (size_t i = 0; i < ngroup1; i++) {
-    _forwardings[j++] = ZForwarding::create(group1[i]);
+  // Populate small pages
+  for (size_t i = 0; i < nsmall; i++) {
+    _forwardings[j++] = ZForwarding::alloc(&_allocator, small[i]);
   }
+
+  assert(_allocator.is_full(), "Should be full");
+
+  // Update statistics
+  ZStatRelocation::set_at_populate_relocation_set(_allocator.size());
 }
 
 void ZRelocationSet::reset() {
-  for (size_t i = 0; i < _nforwardings; i++) {
-    ZForwarding::destroy(_forwardings[i]);
-    _forwardings[i] = NULL;
-  }
+  _nforwardings = 0;
 }

src/hotspot/share/gc/z/zRelocationSet.hpp

@@ -25,6 +25,7 @@
 #define SHARE_GC_Z_ZRELOCATIONSET_HPP
 
 #include "gc/z/zArray.hpp"
+#include "gc/z/zForwardingAllocator.hpp"
 #include "memory/allocation.hpp"
 
 class ZForwarding;
@@ -34,14 +35,16 @@ class ZRelocationSet {
   template <bool> friend class ZRelocationSetIteratorImpl;
 
 private:
-  ZForwarding** _forwardings;
-  size_t        _nforwardings;
+  ZForwardingAllocator _allocator;
+  ZForwarding**        _forwardings;
+  size_t               _nforwardings;
 
 public:
   ZRelocationSet();
 
-  void populate(ZPage* const* group0, size_t ngroup0,
-                ZPage* const* group1, size_t ngroup1);
+  void populate(ZPage* const* small, size_t nsmall,
+                ZPage* const* medium, size_t nmedium,
+                size_t forwarding_entries);
   void reset();
 };
 

src/hotspot/share/gc/z/zRelocationSetSelector.cpp

@@ -23,6 +23,7 @@
 
 #include "precompiled.hpp"
 #include "gc/z/zArray.inline.hpp"
+#include "gc/z/zForwarding.inline.hpp"
 #include "gc/z/zPage.inline.hpp"
 #include "gc/z/zRelocationSet.hpp"
 #include "gc/z/zRelocationSetSelector.inline.hpp"
@@ -53,6 +54,7 @@ ZRelocationSetSelectorGroup::ZRelocationSetSelectorGroup(const char* name,
     _registered_pages(),
     _sorted_pages(NULL),
     _nselected(0),
+    _forwarding_entries(0),
     _stats() {}
 
 ZRelocationSetSelectorGroup::~ZRelocationSetSelectorGroup() {
@@ -143,19 +145,23 @@ void ZRelocationSetSelectorGroup::select_inner() {
   const size_t npages = _registered_pages.length();
   size_t selected_from = 0;
   size_t selected_to = 0;
-  size_t from_size = 0;
+  size_t selected_forwarding_entries = 0;
+  size_t from_live_bytes = 0;
+  size_t from_forwarding_entries = 0;
 
   semi_sort();
 
   for (size_t from = 1; from <= npages; from++) {
     // Add page to the candidate relocation set
-    from_size += _sorted_pages[from - 1]->live_bytes();
+    ZPage* const page = _sorted_pages[from - 1];
+    from_live_bytes += page->live_bytes();
+    from_forwarding_entries += ZForwarding::nentries(page);
 
     // Calculate the maximum number of pages needed by the candidate relocation set.
     // By subtracting the object size limit from the pages size we get the maximum
     // number of pages that the relocation set is guaranteed to fit in, regardless
     // of in which order the objects are relocated.
-    const size_t to = ceil((double)(from_size) / (double)(_page_size - _object_size_limit));
+    const size_t to = ceil((double)(from_live_bytes) / (double)(_page_size - _object_size_limit));
 
     // Calculate the relative difference in reclaimable space compared to our
     // currently selected final relocation set. If this number is larger than the
@@ -167,22 +173,27 @@ void ZRelocationSetSelectorGroup::select_inner() {
     if (diff_reclaimable > ZFragmentationLimit) {
       selected_from = from;
       selected_to = to;
+      selected_forwarding_entries = from_forwarding_entries;
     }
 
-    log_trace(gc, reloc)("Candidate Relocation Set (%s Pages): "
-                         SIZE_FORMAT "->" SIZE_FORMAT ", %.1f%% relative defragmentation, %s",
-                         _name, from, to, diff_reclaimable, (selected_from == from) ? "Selected" : "Rejected");
+    log_trace(gc, reloc)("Candidate Relocation Set (%s Pages): " SIZE_FORMAT "->" SIZE_FORMAT ", "
+                         "%.1f%% relative defragmentation, " SIZE_FORMAT " forwarding entries, %s",
+                         _name, from, to, diff_reclaimable, from_forwarding_entries,
+                         (selected_from == from) ? "Selected" : "Rejected");
   }
 
   // Finalize selection
   _nselected = selected_from;
+  _forwarding_entries = selected_forwarding_entries;
 
   // Update statistics
   _stats._compacting_from = selected_from * _page_size;
   _stats._compacting_to = selected_to * _page_size;
 
-  log_trace(gc, reloc)("Relocation Set (%s Pages): " SIZE_FORMAT "->" SIZE_FORMAT ", " SIZE_FORMAT " skipped",
-                       _name, selected_from, selected_to, npages - _nselected);
+  log_trace(gc, reloc)("Relocation Set (%s Pages): " SIZE_FORMAT "->" SIZE_FORMAT ", "
+                       SIZE_FORMAT " skipped, " SIZE_FORMAT " forwarding entries",
+                       _name, selected_from, selected_to, npages - selected_from,
+                       selected_forwarding_entries);
 }
 
 void ZRelocationSetSelectorGroup::select() {
@@ -245,8 +256,9 @@ void ZRelocationSetSelector::select(ZRelocationSet* relocation_set) {
   _small.select();
 
   // Populate relocation set
-  relocation_set->populate(_medium.selected(), _medium.nselected(),
-                           _small.selected(), _small.nselected());
+  relocation_set->populate(_small.selected(), _small.nselected(),
+                           _medium.selected(), _medium.nselected(),
+                           forwarding_entries());
 
   // Send event
   event.commit(total(), empty(), compacting_from(), compacting_to());

src/hotspot/share/gc/z/zRelocationSetSelector.hpp

@@ -75,10 +75,10 @@ private:
   const size_t                     _page_size;
   const size_t                     _object_size_limit;
   const size_t                     _fragmentation_limit;
-
   ZArray<ZPage*>                   _registered_pages;
   ZPage**                          _sorted_pages;
   size_t                           _nselected;
+  size_t                           _forwarding_entries;
   ZRelocationSetSelectorGroupStats _stats;
 
   bool is_disabled();
@@ -99,6 +99,7 @@ public:
 
   ZPage* const* selected() const;
   size_t nselected() const;
+  size_t forwarding_entries() const;
 
   const ZRelocationSetSelectorGroupStats& stats() const;
 };
@@ -109,6 +110,7 @@ private:
   ZRelocationSetSelectorGroup _medium;
   ZRelocationSetSelectorGroup _large;
 
+  size_t forwarding_entries() const;
   size_t total() const;
   size_t empty() const;
   size_t compacting_from() const;

src/hotspot/share/gc/z/zRelocationSetSelector.inline.hpp

@@ -74,10 +74,18 @@ inline size_t ZRelocationSetSelectorGroup::nselected() const {
   return _nselected;
 }
 
+inline size_t ZRelocationSetSelectorGroup::forwarding_entries() const {
+  return _forwarding_entries;
+}
+
 inline const ZRelocationSetSelectorGroupStats& ZRelocationSetSelectorGroup::stats() const {
   return _stats;
 }
 
+inline size_t ZRelocationSetSelector::forwarding_entries() const {
+  return _small.forwarding_entries() + _medium.forwarding_entries();
+}
+
 inline size_t ZRelocationSetSelector::total() const {
   return _small.stats().total() + _medium.stats().total() + _large.stats().total();
 }

src/hotspot/share/gc/z/zStat.cpp

@@ -1140,12 +1140,17 @@ void ZStatMark::print() {
 // Stat relocation
 //
 ZRelocationSetSelectorStats ZStatRelocation::_stats;
+size_t                      ZStatRelocation::_forwarding_usage;
 bool                        ZStatRelocation::_success;
 
 void ZStatRelocation::set_at_select_relocation_set(const ZRelocationSetSelectorStats& stats) {
   _stats = stats;
 }
 
+void ZStatRelocation::set_at_populate_relocation_set(size_t forwarding_usage) {
+  _forwarding_usage = forwarding_usage;
+}
+
 void ZStatRelocation::set_at_relocate_end(bool success) {
   _success = success;
 }
@@ -1169,6 +1174,7 @@ void ZStatRelocation::print() {
   }
   print("Large", _stats.large());
 
+  log_info(gc, reloc)("Forwarding Usage: " SIZE_FORMAT "M", _forwarding_usage / M);
   log_info(gc, reloc)("Relocation: %s", _success ? "Successful" : "Incomplete");
 }
 

src/hotspot/share/gc/z/zStat.hpp

@@ -423,12 +423,14 @@ public:
 class ZStatRelocation : public AllStatic {
 private:
   static ZRelocationSetSelectorStats _stats;
+  static size_t                      _forwarding_usage;
   static bool                        _success;
 
   static void print(const char* name, const ZRelocationSetSelectorGroupStats& group);
 
 public:
   static void set_at_select_relocation_set(const ZRelocationSetSelectorStats& stats);
+  static void set_at_populate_relocation_set(size_t forwarding_usage);
   static void set_at_relocate_end(bool success);
 
   static void print();

test/hotspot/gtest/gc/z/test_zForwarding.cpp

@@ -24,6 +24,7 @@
 #include "precompiled.hpp"
 #include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zForwarding.inline.hpp"
+#include "gc/z/zForwardingAllocator.inline.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zPage.inline.hpp"
 #include "unittest.hpp"
@@ -157,14 +158,16 @@ public:
     const size_t live_bytes = live_objects * object_size;
     page.inc_live(live_objects, live_bytes);
 
+    // Setup allocator
+    ZForwardingAllocator allocator;
+    const uint32_t nentries = ZForwarding::nentries(&page);
+    allocator.reset((sizeof(ZForwarding)) + (nentries * sizeof(ZForwardingEntry)));
+
     // Setup forwarding
-    ZForwarding* const forwarding = ZForwarding::create(&page);
+    ZForwarding* const forwarding = ZForwarding::alloc(&allocator, &page);
 
     // Actual test function
     (*function)(forwarding);
-
-    // Teardown forwarding
-    ZForwarding::destroy(forwarding);
   }
 
   // Run the given function with a few different input values.