8376822: UseCompactObjectHeaders: fill Klass alignment gaps in AOT cache

Reviewed-by: jsjolen, asmehra

src/hotspot/share/cds/aotMapLogger.cpp

@@ -98,8 +98,8 @@ void AOTMapLogger::dumptime_log(ArchiveBuilder* builder, FileMapInfo* mapinfo,
   DumpRegion* rw_region = &builder->_rw_region;
   DumpRegion* ro_region = &builder->_ro_region;
 
-  dumptime_log_metaspace_region("rw region", rw_region, &builder->_rw_src_objs);
-  dumptime_log_metaspace_region("ro region", ro_region, &builder->_ro_src_objs);
+  dumptime_log_metaspace_region("rw region", rw_region, &builder->_rw_src_objs, &builder->_ro_src_objs);
+  dumptime_log_metaspace_region("ro region", ro_region, &builder->_rw_src_objs, &builder->_ro_src_objs);
 
   address bitmap_end = address(bitmap + bitmap_size_in_bytes);
   log_region_range("bitmap", address(bitmap), bitmap_end, nullptr);
@@ -122,17 +122,6 @@ void AOTMapLogger::dumptime_log(ArchiveBuilder* builder, FileMapInfo* mapinfo,
 class AOTMapLogger::RuntimeGatherArchivedMetaspaceObjs : public UniqueMetaspaceClosure {
   GrowableArrayCHeap<ArchivedObjInfo, mtClass> _objs;
 
-  static int compare_objs_by_addr(ArchivedObjInfo* a, ArchivedObjInfo* b) {
-    intx diff = a->_src_addr - b->_src_addr;
-    if (diff < 0) {
-      return -1;
-    } else if (diff == 0) {
-      return 0;
-    } else {
-      return 1;
-    }
-  }
-
 public:
   GrowableArrayCHeap<ArchivedObjInfo, mtClass>* objs() { return &_objs; }
 
@@ -152,7 +141,7 @@ public:
 
   void finish() {
     UniqueMetaspaceClosure::finish();
-    _objs.sort(compare_objs_by_addr);
+    _objs.sort(compare_by_address);
   }
 }; // AOTMapLogger::RuntimeGatherArchivedMetaspaceObjs
 
@@ -203,24 +192,47 @@ void AOTMapLogger::runtime_log(FileMapInfo* mapinfo, GrowableArrayCHeap<Archived
 }
 
 void AOTMapLogger::dumptime_log_metaspace_region(const char* name, DumpRegion* region,
-                                                 const ArchiveBuilder::SourceObjList* src_objs) {
+                                                 const ArchiveBuilder::SourceObjList* rw_objs,
+                                                 const ArchiveBuilder::SourceObjList* ro_objs) {
   address region_base = address(region->base());
   address region_top  = address(region->top());
   log_region_range(name, region_base, region_top, region_base + _buffer_to_requested_delta);
   if (log_is_enabled(Debug, aot, map)) {
     GrowableArrayCHeap<ArchivedObjInfo, mtClass> objs;
-    for (int i = 0; i < src_objs->objs()->length(); i++) {
-      ArchiveBuilder::SourceObjInfo* src_info = src_objs->at(i);
+    // With -XX:+UseCompactObjectHeaders, it's possible for small objects (including some from
+    // ro_objs) to be allocated in the gaps in the RW region.
+    collect_metaspace_objs(&objs, region_base, region_top, rw_objs);
+    collect_metaspace_objs(&objs, region_base, region_top, ro_objs);
+    objs.sort(compare_by_address);
+    log_metaspace_objects_impl(address(region->base()), address(region->end()), &objs, 0, objs.length());
+  }
+}
+
+void AOTMapLogger::collect_metaspace_objs(GrowableArrayCHeap<ArchivedObjInfo, mtClass>* objs,
+                                          address region_base, address region_top ,
+                                          const ArchiveBuilder::SourceObjList* src_objs) {
+  for (int i = 0; i < src_objs->objs()->length(); i++) {
+    ArchiveBuilder::SourceObjInfo* src_info = src_objs->at(i);
+    address buf_addr = src_info->buffered_addr();
+    if (region_base <= buf_addr && buf_addr < region_top) {
       ArchivedObjInfo info;
       info._src_addr = src_info->source_addr();
-      info._buffered_addr = src_info->buffered_addr();
+      info._buffered_addr = buf_addr;
       info._requested_addr = info._buffered_addr + _buffer_to_requested_delta;
       info._bytes = src_info->size_in_bytes();
       info._type = src_info->type();
-      objs.append(info);
+      objs->append(info);
     }
+  }
+}
 
-    log_metaspace_objects_impl(address(region->base()), address(region->end()), &objs, 0, objs.length());
+int AOTMapLogger::compare_by_address(ArchivedObjInfo* a, ArchivedObjInfo* b) {
+  if (a->_buffered_addr < b->_buffered_addr) {
+    return -1;
+  } else if (a->_buffered_addr > b->_buffered_addr) {
+    return 1;
+  } else {
+    return 0;
   }
 }
 

src/hotspot/share/cds/aotMapLogger.hpp

@@ -127,7 +127,12 @@ private:
   static void runtime_log(FileMapInfo* mapinfo, GrowableArrayCHeap<ArchivedObjInfo, mtClass>* objs);
   static void runtime_log_metaspace_regions(FileMapInfo* mapinfo, GrowableArrayCHeap<ArchivedObjInfo, mtClass>* objs);
   static void dumptime_log_metaspace_region(const char* name, DumpRegion* region,
-                                            const ArchiveBuilder::SourceObjList* src_objs);
+                                            const ArchiveBuilder::SourceObjList* rw_objs,
+                                            const ArchiveBuilder::SourceObjList* ro_objs);
+  static void collect_metaspace_objs(GrowableArrayCHeap<ArchivedObjInfo, mtClass>* objs,
+                                     address region_base, address region_top ,
+                                     const ArchiveBuilder::SourceObjList* src_objs);
+  static int compare_by_address(ArchivedObjInfo* a, ArchivedObjInfo* b);
 
   // Common code for dumptime/runtime
   static void log_file_header(FileMapInfo* mapinfo);

src/hotspot/share/cds/archiveBuilder.cpp

@@ -627,6 +627,7 @@ void ArchiveBuilder::dump_ro_metadata() {
   start_dump_region(&_ro_region);
   make_shallow_copies(&_ro_region, &_ro_src_objs);
   RegeneratedClasses::record_regenerated_objects();
+  DumpRegion::report_gaps(&_alloc_stats);
 }
 
 void ArchiveBuilder::make_shallow_copies(DumpRegion *dump_region,
@@ -639,33 +640,10 @@ void ArchiveBuilder::make_shallow_copies(DumpRegion *dump_region,
 
 void ArchiveBuilder::make_shallow_copy(DumpRegion *dump_region, SourceObjInfo* src_info) {
   address src = src_info->source_addr();
-  int bytes = src_info->size_in_bytes(); // word-aligned
-  size_t alignment = SharedSpaceObjectAlignment; // alignment for the dest pointer
+  int bytes = src_info->size_in_bytes();
+  char* dest = dump_region->allocate_metaspace_obj(bytes, src, src_info->type(),
+                                                   src_info->read_only(), &_alloc_stats);
 
-  char* oldtop = dump_region->top();
-  if (src_info->type() == MetaspaceClosureType::ClassType) {
-    // Allocate space for a pointer directly in front of the future InstanceKlass, so
-    // we can do a quick lookup from InstanceKlass* -> RunTimeClassInfo*
-    // without building another hashtable. See RunTimeClassInfo::get_for()
-    // in systemDictionaryShared.cpp.
-    Klass* klass = (Klass*)src;
-    if (klass->is_instance_klass()) {
-      SystemDictionaryShared::validate_before_archiving(InstanceKlass::cast(klass));
-      dump_region->allocate(sizeof(address));
-    }
-#ifdef _LP64
-    // More strict alignments needed for UseCompressedClassPointers
-    if (UseCompressedClassPointers) {
-      alignment = nth_bit(ArchiveBuilder::precomputed_narrow_klass_shift());
-    }
-#endif
-  } else if (src_info->type() == MetaspaceClosureType::SymbolType) {
-    // Symbols may be allocated by using AllocateHeap, so their sizes
-    // may be less than size_in_bytes() indicates.
-    bytes = ((Symbol*)src)->byte_size();
-  }
-
-  char* dest = dump_region->allocate(bytes, alignment);
   memcpy(dest, src, bytes);
 
   // Update the hash of buffered sorted symbols for static dump so that the symbols have deterministic contents
@@ -692,11 +670,6 @@ void ArchiveBuilder::make_shallow_copy(DumpRegion *dump_region, SourceObjInfo* s
 
   log_trace(aot)("Copy: " PTR_FORMAT " ==> " PTR_FORMAT " %d", p2i(src), p2i(dest), bytes);
   src_info->set_buffered_addr((address)dest);
-
-  char* newtop = dump_region->top();
-  _alloc_stats.record(src_info->type(), int(newtop - oldtop), src_info->read_only());
-
-  DEBUG_ONLY(_alloc_stats.verify((int)dump_region->used(), src_info->read_only()));
 }
 
 // This is used by code that hand-assembles data structures, such as the LambdaProxyClassKey, that are

src/hotspot/share/cds/archiveUtils.cpp

@@ -30,6 +30,7 @@
 #include "cds/cdsConfig.hpp"
 #include "cds/classListParser.hpp"
 #include "cds/classListWriter.hpp"
+#include "cds/dumpAllocStats.hpp"
 #include "cds/dynamicArchive.hpp"
 #include "cds/filemap.hpp"
 #include "cds/heapShared.hpp"
@@ -46,6 +47,7 @@
 #include "utilities/debug.hpp"
 #include "utilities/formatBuffer.hpp"
 #include "utilities/globalDefinitions.hpp"
+#include "utilities/rbTree.inline.hpp"
 #include "utilities/spinYield.hpp"
 
 CHeapBitMap* ArchivePtrMarker::_ptrmap = nullptr;
@@ -116,13 +118,17 @@ void ArchivePtrMarker::mark_pointer(address* ptr_loc) {
 
   if (ptr_base() <= ptr_loc && ptr_loc < ptr_end()) {
     address value = *ptr_loc;
-    // We don't want any pointer that points to very bottom of the archive, otherwise when
-    // AOTMetaspace::default_base_address()==0, we can't distinguish between a pointer
-    // to nothing (null) vs a pointer to an objects that happens to be at the very bottom
-    // of the archive.
-    assert(value != (address)ptr_base(), "don't point to the bottom of the archive");
-
     if (value != nullptr) {
+      // We don't want any pointer that points to very bottom of the AOT metaspace, otherwise
+      // when AOTMetaspace::default_base_address()==0, we can't distinguish between a pointer
+      // to nothing (null) vs a pointer to an objects that happens to be at the very bottom
+      // of the AOT metaspace.
+      //
+      // This should never happen because the protection zone prevents any valid objects from
+      // being allocated at the bottom of the AOT metaspace.
+      assert(AOTMetaspace::protection_zone_size() > 0, "must be");
+      assert(ArchiveBuilder::current()->any_to_offset(value) > 0, "cannot point to bottom of AOT metaspace");
+
       assert(uintx(ptr_loc) % sizeof(intptr_t) == 0, "pointers must be stored in aligned addresses");
       size_t idx = ptr_loc - ptr_base();
       if (_ptrmap->size() <= idx) {
@@ -130,7 +136,6 @@ void ArchivePtrMarker::mark_pointer(address* ptr_loc) {
       }
       assert(idx < _ptrmap->size(), "must be");
       _ptrmap->set_bit(idx);
-      //tty->print_cr("Marking pointer [" PTR_FORMAT "] -> " PTR_FORMAT " @ %5zu", p2i(ptr_loc), p2i(*ptr_loc), idx);
     }
   }
 }
@@ -144,7 +149,6 @@ void ArchivePtrMarker::clear_pointer(address* ptr_loc) {
   size_t idx = ptr_loc - ptr_base();
   assert(idx < _ptrmap->size(), "cannot clear pointers that have not been marked");
   _ptrmap->clear_bit(idx);
-  //tty->print_cr("Clearing pointer [" PTR_FORMAT "] -> " PTR_FORMAT " @ %5zu", p2i(ptr_loc), p2i(*ptr_loc), idx);
 }
 
 class ArchivePtrBitmapCleaner: public BitMapClosure {
@@ -249,16 +253,179 @@ void DumpRegion::commit_to(char* newtop) {
                  which, commit, _vs->actual_committed_size(), _vs->high());
 }
 
+// Basic allocation. Any alignment gaps will be wasted.
 char* DumpRegion::allocate(size_t num_bytes, size_t alignment) {
   // Always align to at least minimum alignment
   alignment = MAX2(SharedSpaceObjectAlignment, alignment);
   char* p = (char*)align_up(_top, alignment);
-  char* newtop = p + align_up(num_bytes, (size_t)SharedSpaceObjectAlignment);
+  char* newtop = p + align_up(num_bytes, SharedSpaceObjectAlignment);
   expand_top_to(newtop);
   memset(p, 0, newtop - p);
   return p;
 }
 
+class DumpRegion::AllocGap {
+  size_t _gap_bytes;   // size of this gap in bytes
+  char* _gap_bottom;   // must be SharedSpaceObjectAlignment aligned
+public:
+  size_t gap_bytes() const { return _gap_bytes; }
+  char* gap_bottom() const { return _gap_bottom; }
+
+  AllocGap(size_t bytes, char* bottom) : _gap_bytes(bytes), _gap_bottom(bottom) {
+    precond(is_aligned(gap_bytes(), SharedSpaceObjectAlignment));
+    precond(is_aligned(gap_bottom(), SharedSpaceObjectAlignment));
+  }
+};
+
+struct DumpRegion::AllocGapCmp {
+  static RBTreeOrdering cmp(AllocGap a, AllocGap b) {
+    RBTreeOrdering order = rbtree_primitive_cmp(a.gap_bytes(), b.gap_bytes());
+    if (order == RBTreeOrdering::EQ) {
+      order = rbtree_primitive_cmp(a.gap_bottom(), b.gap_bottom());
+    }
+    return order;
+  }
+};
+
+struct Empty {};
+using AllocGapNode = RBNode<DumpRegion::AllocGap, Empty>;
+
+class DumpRegion::AllocGapTree : public RBTreeCHeap<AllocGap, Empty, AllocGapCmp, mtClassShared> {
+public:
+  size_t add_gap(char* gap_bottom, char* gap_top) {
+    precond(gap_bottom < gap_top);
+    size_t gap_bytes = pointer_delta(gap_top, gap_bottom, 1);
+    precond(gap_bytes > 0);
+
+    _total_gap_bytes += gap_bytes;
+
+    AllocGap gap(gap_bytes, gap_bottom); // constructor checks alignment
+    AllocGapNode* node = allocate_node(gap, Empty{});
+    insert(gap, node);
+
+    log_trace(aot, alloc)("adding a gap of %zu bytes @ %p (total = %zu) in %zu blocks", gap_bytes, gap_bottom, _total_gap_bytes, size());
+    return gap_bytes;
+  }
+
+  char* allocate_from_gap(size_t num_bytes) {
+    // The gaps are sorted in ascending order of their sizes. When two gaps have the same
+    // size, the one with a lower gap_bottom comes first.
+    //
+    // Find the first gap that's big enough, with the lowest gap_bottom.
+    AllocGap target(num_bytes, nullptr);
+    AllocGapNode* node = closest_ge(target);
+    if (node == nullptr) {
+      return nullptr; // Didn't find any usable gap.
+    }
+
+    size_t gap_bytes = node->key().gap_bytes();
+    char* gap_bottom = node->key().gap_bottom();
+    char* result = gap_bottom;
+    precond(is_aligned(result, SharedSpaceObjectAlignment));
+
+    remove(node);
+
+    precond(_total_gap_bytes >= num_bytes);
+    _total_gap_bytes -= num_bytes;
+    _total_gap_bytes_used += num_bytes;
+    _total_gap_allocs++;
+    DEBUG_ONLY(node = nullptr); // Don't use it anymore!
+
+    precond(gap_bytes >= num_bytes);
+    if (gap_bytes > num_bytes) {
+      gap_bytes -= num_bytes;
+      gap_bottom += num_bytes;
+
+      AllocGap gap(gap_bytes, gap_bottom); // constructor checks alignment
+      AllocGapNode* new_node = allocate_node(gap, Empty{});
+      insert(gap, new_node);
+    }
+    log_trace(aot, alloc)("%zu bytes @ %p in a gap of %zu bytes (used gaps %zu times, remain gap = %zu bytes in %zu blocks)",
+                          num_bytes, result, gap_bytes, _total_gap_allocs, _total_gap_bytes, size());
+    return result;
+  }
+};
+
+size_t DumpRegion::_total_gap_bytes = 0;
+size_t DumpRegion::_total_gap_bytes_used = 0;
+size_t DumpRegion::_total_gap_allocs = 0;
+DumpRegion::AllocGapTree DumpRegion::_gap_tree;
+
+// Alignment gaps happen only for the RW space. Collect the gaps into the _gap_tree so they can be
+// used for future small object allocation.
+char* DumpRegion::allocate_metaspace_obj(size_t num_bytes, address src, MetaspaceClosureType type, bool read_only, DumpAllocStats* stats) {
+  num_bytes = align_up(num_bytes, SharedSpaceObjectAlignment);
+  size_t alignment = SharedSpaceObjectAlignment; // alignment for the dest pointer
+  bool is_class = (type == MetaspaceClosureType::ClassType);
+  bool is_instance_class = is_class && ((Klass*)src)->is_instance_klass();
+
+#ifdef _LP64
+  // More strict alignments needed for UseCompressedClassPointers
+  if (is_class && UseCompressedClassPointers) {
+    size_t klass_alignment = checked_cast<size_t>(nth_bit(ArchiveBuilder::precomputed_narrow_klass_shift()));
+    alignment = MAX2(alignment, klass_alignment);
+    precond(is_aligned(alignment, SharedSpaceObjectAlignment));
+  }
+#endif
+
+  if (alignment == SharedSpaceObjectAlignment && type != MetaspaceClosureType::SymbolType) {
+    // The addresses of Symbols must be in the same order as they are in ArchiveBuilder::SourceObjList.
+    // If we put them in gaps, their order will change.
+    //
+    // We have enough small objects that all gaps are usually filled.
+    char* p = _gap_tree.allocate_from_gap(num_bytes);
+    if (p != nullptr) {
+      // Already memset to 0 when adding the gap
+      stats->record(type, checked_cast<int>(num_bytes), /*read_only=*/false); // all gaps are from RW space (for classes)
+      return p;
+    }
+  }
+
+  // Reserve space for a pointer directly in front of the buffered InstanceKlass, so
+  // we can do a quick lookup from InstanceKlass* -> RunTimeClassInfo*
+  // without building another hashtable. See RunTimeClassInfo::get_for()
+  // in systemDictionaryShared.cpp.
+  const size_t RuntimeClassInfoPtrSize = is_instance_class ? sizeof(address) : 0;
+
+  if (is_class && !is_aligned(top() + RuntimeClassInfoPtrSize, alignment)) {
+    // We need to add a gap to align the buffered Klass. Save the gap for future small allocations.
+    assert(read_only == false, "only gaps in RW region are reusable");
+    char* gap_bottom = top();
+    char* gap_top = align_up(gap_bottom + RuntimeClassInfoPtrSize, alignment) - RuntimeClassInfoPtrSize;
+    size_t gap_bytes = _gap_tree.add_gap(gap_bottom, gap_top);
+    allocate(gap_bytes);
+  }
+
+  char* oldtop = top();
+  if (is_instance_class) {
+    SystemDictionaryShared::validate_before_archiving((InstanceKlass*)src);
+    allocate(RuntimeClassInfoPtrSize);
+  }
+
+  precond(is_aligned(top(), alignment));
+  char* result = allocate(num_bytes);
+  log_trace(aot, alloc)("%zu bytes @ %p", num_bytes, result);
+  stats->record(type, pointer_delta_as_int(top(), oldtop), read_only); // includes RuntimeClassInfoPtrSize for classes
+
+  return result;
+}
+
+// Usually we have no gaps left.
+void DumpRegion::report_gaps(DumpAllocStats* stats) {
+  _gap_tree.visit_in_order([&](const AllocGapNode* node) {
+        stats->record_gap(checked_cast<int>(node->key().gap_bytes()));
+        return true;
+      });
+  if (_gap_tree.size() > 0) {
+    log_warning(aot)("Unexpected %zu gaps (%zu bytes) for Klass alignment",
+                     _gap_tree.size(), _total_gap_bytes);
+  }
+  if (_total_gap_allocs > 0) {
+    log_info(aot)("Allocated %zu objects of %zu bytes in gaps (remain = %zu bytes)",
+                  _total_gap_allocs, _total_gap_bytes_used, _total_gap_bytes);
+  }
+}
+
 void DumpRegion::append_intptr_t(intptr_t n, bool need_to_mark) {
   assert(is_aligned(_top, sizeof(intptr_t)), "bad alignment");
   intptr_t *p = (intptr_t*)_top;

src/hotspot/share/cds/archiveUtils.hpp

@@ -28,7 +28,9 @@
 #include "cds/cds_globals.hpp"
 #include "cds/serializeClosure.hpp"
 #include "logging/log.hpp"
+#include "memory/allocation.hpp"
 #include "memory/metaspace.hpp"
+#include "memory/metaspaceClosureType.hpp"
 #include "memory/virtualspace.hpp"
 #include "runtime/nonJavaThread.hpp"
 #include "runtime/semaphore.hpp"
@@ -37,6 +39,7 @@
 #include "utilities/macros.hpp"
 
 class BootstrapInfo;
+class DumpAllocStats;
 class ReservedSpace;
 class VirtualSpace;
 
@@ -159,6 +162,18 @@ private:
 
   void commit_to(char* newtop);
 
+public:
+  // Allocation gaps (due to Klass alignment)
+  class AllocGapTree;
+  class AllocGap;
+  struct AllocGapCmp;
+
+private:
+  static AllocGapTree _gap_tree;
+  static size_t _total_gap_bytes;
+  static size_t _total_gap_bytes_used;
+  static size_t _total_gap_allocs;
+
 public:
   DumpRegion(const char* name)
     : _name(name), _base(nullptr), _top(nullptr), _end(nullptr),
@@ -167,6 +182,7 @@ public:
 
   char* expand_top_to(char* newtop);
   char* allocate(size_t num_bytes, size_t alignment = 0);
+  char* allocate_metaspace_obj(size_t num_bytes, address src, MetaspaceClosureType type, bool read_only, DumpAllocStats* stats);
 
   void append_intptr_t(intptr_t n, bool need_to_mark = false) NOT_CDS_RETURN;
 
@@ -191,6 +207,8 @@ public:
   bool contains(char* p) {
     return base() <= p && p < top();
   }
+
+  static void report_gaps(DumpAllocStats* stats);
 };
 
 // Closure for serializing initialization data out to a data area to be

src/hotspot/share/cds/dumpAllocStats.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -129,15 +129,3 @@ void DumpAllocStats::print_stats(int ro_all, int rw_all) {
                                                                   _bytes [RW][MethodTrainingDataType]);
 
 }
-
-#ifdef ASSERT
-void DumpAllocStats::verify(int expected_byte_size, bool read_only) const {
-  int bytes = 0;
-  const int what = (int)(read_only ? RO : RW);
-  for (int type = 0; type < int(_number_of_types); type ++) {
-    bytes += _bytes[what][type];
-  }
-  assert(bytes == expected_byte_size, "counter mismatch (%s: %d vs %d)",
-         (read_only ? "RO" : "RW"), bytes, expected_byte_size);
-}
-#endif // ASSERT

src/hotspot/share/cds/dumpAllocStats.hpp

@@ -41,6 +41,7 @@ public:
   f(StringHashentry) \
   f(StringBucket) \
   f(CppVTables) \
+  f(Gap) \
   f(Other)
 
 #define DUMPED_TYPE_DECLARE(name) name ## Type,
@@ -111,12 +112,19 @@ public:
     _bytes [which][t] += byte_size;
   }
 
+  void record_gap(int byte_size) {
+    _counts[RW][GapType] += 1;
+    _bytes [RW][GapType] += byte_size;
+  }
+
   void record_other_type(int byte_size, bool read_only) {
     int which = (read_only) ? RO : RW;
+    _counts[which][OtherType] += 1;
     _bytes [which][OtherType] += byte_size;
   }
 
   void record_cpp_vtables(int byte_size) {
+    _counts[RW][CppVTablesType] += 1;
     _bytes[RW][CppVTablesType] += byte_size;
   }
 
@@ -145,9 +153,6 @@ public:
   }
 
   void print_stats(int ro_all, int rw_all);
-
-  DEBUG_ONLY(void verify(int expected_byte_size, bool read_only) const);
-
 };
 
 #endif // SHARE_CDS_DUMPALLOCSTATS_HPP

src/hotspot/share/utilities/rbTree.hpp

@@ -429,7 +429,12 @@ public:
   void free(void* ptr);
 };
 
-
+template <typename T>
+RBTreeOrdering rbtree_primitive_cmp(T a, T b) { // handy function
+  if (a < b) return RBTreeOrdering::LT;
+  if (a > b) return RBTreeOrdering::GT;
+  return RBTreeOrdering::EQ;
+}
 
 template <typename K, typename V, typename COMPARATOR, MemTag mem_tag, AllocFailType strategy = AllocFailStrategy::EXIT_OOM>
 using RBTreeCHeap = RBTree<K, V, COMPARATOR, RBTreeCHeapAllocator<mem_tag, strategy>>;

test/hotspot/jtreg/runtime/cds/MetaspaceAllocGaps.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @requires vm.cds
+ * @requires vm.flagless
+ * @requires vm.bits == 64
+ * @bug 8376822
+ * @summary Allocation gaps in the RW region caused by -XX:+UseCompactObjectHeaders should be reused
+ * @library /test/lib
+ * @build MetaspaceAllocGaps
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller -jar hello.jar Hello
+ * @run driver MetaspaceAllocGaps
+ */
+
+import jdk.test.lib.cds.SimpleCDSAppTester;
+import jdk.test.lib.helpers.ClassFileInstaller;
+import jdk.test.lib.process.OutputAnalyzer;
+
+public class MetaspaceAllocGaps {
+    public static void main(String[] args) throws Exception {
+        String appJar = ClassFileInstaller.getJarPath("hello.jar");
+        for (int i = 0; i < 2; i++) {
+            String compressedOops = "-XX:" + (i == 0 ? "-" : "+") + "UseCompressedOops";
+            SimpleCDSAppTester.of("MetaspaceAllocGaps" + i)
+                .addVmArgs("-Xlog:aot=debug,aot+alloc=trace",
+                           "-XX:+UseCompactObjectHeaders")
+                .classpath(appJar)
+                .appCommandLine("Hello")
+                .setTrainingChecker((OutputAnalyzer out) -> {
+                    // Typically all gaps should be filled. If not, we probably have a regression in C++ class ArchiveUtils.
+                    //
+                    // [0.422s][debug][aot ] Detailed metadata info (excluding heap region):
+                    // [...]
+                    // [0.422s][debug][aot ] Gap : 0 0 0.0 | 0 0 0.0 | 0 0 0.0 <<< look for this pattern
+                    out.shouldMatch("Allocated [1-9][0-9]+ objects of [1-9][0-9]+ bytes in gaps .remain = 0 bytes")
+                       .shouldMatch("debug.* Gap .*0[.]0.*0[.]0.*0[.]0")
+                       .shouldNotMatch("Unexpected .* gaps .* for Klass alignment");
+                })
+                .setProductionChecker((OutputAnalyzer out) -> {
+                    out.shouldContain("HelloWorld");
+                })
+                .runAOTWorkflow();
+        }
+    }
+}
+
+class Hello {
+    public static void main(String[] args) {
+        System.out.println("HelloWorld");
+    }
+}