diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp index 782bb079d07..6f962cce716 100644 --- a/src/hotspot/os/aix/os_aix.cpp +++ b/src/hotspot/os/aix/os_aix.cpp @@ -1810,11 +1810,11 @@ os::PlaceholderRegion os::pd_reserve_placeholder_memory(size_t bytes, bool exec, return PlaceholderRegion(); } -os::PlaceholderRegion os::pd_split_memory(PlaceholderRegion& region, size_t offset) { +os::PlaceholderRegionPair os::pd_split_memory(PlaceholderRegion& orig, size_t offset) { // On AIX, mmap regions are inherently splittable. Just do bookkeeping. // pd_reserve_placeholder_memory guarantees mmaped (not shmated) memory. - char* base = region.base(); - size_t region_size = region.size(); + char* base = orig.base(); + size_t region_size = orig.size(); assert(base != nullptr, "Region base cannot be null"); assert(offset > 0, "Offset must be positive"); @@ -1829,11 +1829,7 @@ os::PlaceholderRegion os::pd_split_memory(PlaceholderRegion& region, size_t offs vmi->addr = base + offset; vmi->size = region_size - offset; - // Shrink region to the trailing piece. - region = PlaceholderRegion(base + offset, region_size - offset); - - // Return the leading piece. - return PlaceholderRegion(base, offset); + return {PlaceholderRegion(base, offset), PlaceholderRegion(base + offset, region_size - offset)}; } char* os::pd_convert_to_reserved(PlaceholderRegion region) { diff --git a/src/hotspot/os/posix/os_posix.cpp b/src/hotspot/os/posix/os_posix.cpp index 1c673f26b32..203df878055 100644 --- a/src/hotspot/os/posix/os_posix.cpp +++ b/src/hotspot/os/posix/os_posix.cpp @@ -555,20 +555,16 @@ os::PlaceholderRegion os::pd_reserve_placeholder_memory(size_t bytes, bool exec, return PlaceholderRegion(base, base != nullptr ? bytes : 0); } -os::PlaceholderRegion os::pd_split_memory(PlaceholderRegion& region, size_t offset) { +os::PlaceholderRegionPair os::pd_split_memory(const PlaceholderRegion& orig, size_t offset) { // On POSIX, mmap regions are inherently splittable. Just do bookkeeping. - char* base = region.base(); - size_t region_size = region.size(); + char* base = orig.base(); + size_t region_size = orig.size(); assert(base != nullptr, "Region base cannot be null"); assert(offset > 0, "Offset must be positive"); assert(offset < region_size, "Offset must be less than region size"); - // Shrink region to the trailing piece. - region = PlaceholderRegion(base + offset, region_size - offset); - - // Return the leading piece. - return PlaceholderRegion(base, offset); + return {PlaceholderRegion(base, offset), PlaceholderRegion(base + offset, region_size - offset)}; } char* os::pd_convert_to_reserved(PlaceholderRegion region) { diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp index bd83398b3f0..02eb6921c83 100644 --- a/src/hotspot/os/windows/os_windows.cpp +++ b/src/hotspot/os/windows/os_windows.cpp @@ -3488,11 +3488,11 @@ os::PlaceholderRegion os::pd_reserve_placeholder_memory(size_t bytes, bool exec, } } -os::PlaceholderRegion os::pd_split_memory(PlaceholderRegion& region, size_t offset) { +os::PlaceholderRegionPair os::pd_split_memory(const PlaceholderRegion& orig, size_t offset) { guarantee(is_VirtualAlloc2_supported(), "pd_split_memory requires VirtualAlloc2 on Windows."); - char* base = region.base(); - size_t region_size = region.size(); + char* base = orig.base(); + size_t region_size = orig.size(); assert(base != nullptr, "Region base cannot be null."); assert(offset > 0, "Offset must be positive (nothing to split at 0)."); @@ -3514,11 +3514,7 @@ os::PlaceholderRegion os::pd_split_memory(PlaceholderRegion& region, size_t offs log_trace(os)("Split placeholder " RANGE_FORMAT " at offset %zu.", RANGE_FORMAT_ARGS(base, region_size), offset); - // Shrink region to the trailing piece. - region = PlaceholderRegion(base + offset, region_size - offset); - - // Return the leading piece. - return PlaceholderRegion(base, offset); + return {PlaceholderRegion(base, offset), PlaceholderRegion(base + offset, region_size - offset)}; } char* os::pd_convert_to_reserved(PlaceholderRegion region) { @@ -3576,28 +3572,33 @@ char* os::win32::reserve_with_numa_placeholder(char* addr, size_t bytes) { // Reserve the full range as a placeholder. // If we requested an address, pd_reserve_placeholder_memory will obtain it or fail. - PlaceholderRegion remaining = os::pd_reserve_placeholder_memory(bytes, false, addr); - if (remaining.is_empty()) { + PlaceholderRegion whole_range = os::pd_reserve_placeholder_memory(bytes, false, addr); + if (whole_range.is_empty()) { log_warning(os)("Failed to reserve placeholder for NUMA interleaving (" PTR_FORMAT ", %zu).", p2i(addr), bytes); return nullptr; } - char* const base = remaining.base(); - log_trace(os)("Created VirtualAlloc2 NUMA placeholder at " RANGE_FORMAT " (%zu bytes).", RANGE_FORMAT_ARGS(base, bytes), bytes); + char* const whole_range_base = whole_range.base(); + log_trace(os)("Created VirtualAlloc2 NUMA placeholder at " RANGE_FORMAT " (%zu bytes).", RANGE_FORMAT_ARGS(whole_range_base, bytes), bytes); + + char* cur = whole_range_base; + size_t remaining_len = whole_range.size(); int count = 0; const int node_count = numa_node_list_holder.get_count(); - while (!remaining.is_empty()) { - size_t bytes_to_rq = MIN2(remaining.size(), chunk_size - ((size_t)remaining.base() % chunk_size)); - PlaceholderRegion chunk = os::split_memory(remaining, bytes_to_rq); - + while (remaining_len > 0) { + const size_t bytes_to_rq = MIN2(remaining_len, chunk_size - ((size_t)cur % chunk_size)); + PlaceholderRegion remaining(cur, remaining_len); + os::PlaceholderRegionPair split = os::split_memory(remaining, bytes_to_rq); DWORD node = node_count > 0 ? numa_node_list_holder.get_node_list_entry(count % node_count) : 0; // Assign 0 for testing on UMA systems - convert_placeholder_to_reserved(chunk, (int)node); + convert_placeholder_to_reserved(split.left, (int)node); + cur = split.right.base(); + remaining_len = split.right.size(); count++; } - return base; + return whole_range_base; } // Reserve memory at an arbitrary address, only if that area is diff --git a/src/hotspot/share/cds/aotMetaspace.cpp b/src/hotspot/share/cds/aotMetaspace.cpp index 33fd63bde7c..c274bf21d13 100644 --- a/src/hotspot/share/cds/aotMetaspace.cpp +++ b/src/hotspot/share/cds/aotMetaspace.cpp @@ -1974,11 +1974,10 @@ char* AOTMetaspace::reserve_address_space_for_archives(FileMapInfo* static_mapin os::PlaceholderRegion placeholder = os::reserve_placeholder_memory(total_range_size, mtNone, false /* exec */, (char*)base_address); if (!placeholder.is_empty()) { - os::PlaceholderRegion archive_placeholder = os::split_memory(placeholder, ccs_begin_offset); - // placeholder has been shrunk to [base+ccs_begin_offset, end) = class space + os::PlaceholderRegionPair split = os::split_memory(placeholder, ccs_begin_offset); - char* archive_base = os::convert_to_reserved(archive_placeholder); - char* class_base = os::convert_to_reserved(placeholder); + char* archive_base = os::convert_to_reserved(split.left); + char* class_base = os::convert_to_reserved(split.right); archive_space_rs = ReservedSpace(archive_base, ccs_begin_offset, archive_space_alignment, os::vm_page_size(), diff --git a/src/hotspot/share/runtime/os.cpp b/src/hotspot/share/runtime/os.cpp index 1e7c26c160a..d70ab8a0d76 100644 --- a/src/hotspot/share/runtime/os.cpp +++ b/src/hotspot/share/runtime/os.cpp @@ -1974,36 +1974,32 @@ os::PlaceholderRegion os::reserve_placeholder_memory(size_t bytes, MemTag mem_ta return result; } -os::PlaceholderRegion os::split_memory(PlaceholderRegion& region, size_t offset) { - assert(!region.is_empty(), "Region cannot be empty"); +os::PlaceholderRegionPair os::split_memory(const PlaceholderRegion& orig, size_t offset) { + assert(!orig.is_empty(), "Region cannot be empty"); assert(offset > 0, "Offset must be a value greater than 0"); - assert(offset <= region.size(), "Offset must be less than or equal to region size"); - assert(is_aligned(region.base(), os::vm_page_size()), "Region base should be page-aligned"); + assert(offset <= orig.size(), "Offset must be less than or equal to region size"); + assert(is_aligned(orig.base(), os::vm_page_size()), "Region base should be page-aligned"); assert(is_aligned(offset, os::vm_page_size()), "Offset should be page-aligned"); - char* original_base = region.base(); - size_t original_size = region.size(); + char* original_base = orig.base(); + size_t original_size = orig.size(); if (offset == original_size) { - // No split needed. Return the original region. - PlaceholderRegion result = region; - // The trailing piece is empty now. Nothing left. - region = PlaceholderRegion(); log_debug(os, map)("Split memory consumed the whole region: " RANGEFMT, RANGEFMTARGS(original_base, original_size)); - return result; + return { orig, PlaceholderRegion() }; } - PlaceholderRegion leading = pd_split_memory(region, offset); + PlaceholderRegionPair split = pd_split_memory(orig, offset); - if (leading.is_empty()) { + if (split.left.is_empty() || split.right.is_empty()) { fatal("Split memory at offset %zu failed. Region: " RANGEFMT, offset, RANGEFMTARGS(original_base, original_size)); } log_debug(os, map)("Split memory at offset %zu: " RANGEFMT " -> " RANGEFMT " + " RANGEFMT, offset, RANGEFMTARGS(original_base, original_size), - RANGEFMTARGS(leading.base(), leading.size()), - RANGEFMTARGS(region.base(), region.size())); - return leading; + RANGEFMTARGS(split.left.base(), split.left.size()), + RANGEFMTARGS(split.right.base(), split.right.size())); + return split; } char* os::convert_to_reserved(PlaceholderRegion region) { diff --git a/src/hotspot/share/runtime/os.hpp b/src/hotspot/share/runtime/os.hpp index 29dcacd30f0..07056da09f8 100644 --- a/src/hotspot/share/runtime/os.hpp +++ b/src/hotspot/share/runtime/os.hpp @@ -215,16 +215,22 @@ class os: AllStatic { // MEM_RESERVE_PLACEHOLDER). On POSIX platforms, any mmap'd region is // inherently splittable, so this is a thin wrapper. class PlaceholderRegion { - char* _base; - size_t _size; + char* const _base; + size_t const _size; public: PlaceholderRegion() : _base(nullptr), _size(0) {} PlaceholderRegion(char* base, size_t size) : _base(base), _size(size) {} + PlaceholderRegion(const PlaceholderRegion& source) : _base(source._base), _size(source._size) {} char* base() const { return _base; } size_t size() const { return _size; } bool is_empty() const { return _base == nullptr; } }; + struct PlaceholderRegionPair { + PlaceholderRegion left; + PlaceholderRegion right; + }; + private: static OSThread* _starting_thread; static PageSizes _page_sizes; @@ -243,10 +249,8 @@ class os: AllStatic { static PlaceholderRegion pd_reserve_placeholder_memory(size_t bytes, bool executable, char* addr = nullptr); // On Windows, splits the placeholder with VirtualFree(MEM_PRESERVE_PLACEHOLDER). - // On POSIX, this just does bookkeeping (updates fields of PlaceholderRegion). - // Returns the leading piece [base, base+offset). Shrinks 'region' to become the - // trailing piece [base+offset, base+original_size). - static PlaceholderRegion pd_split_memory(PlaceholderRegion& region, size_t offset); + // On POSIX/AIX, bookkeeping only (AIX updates vmembk). 'orig' must not be reused after this call. + static PlaceholderRegionPair pd_split_memory(const PlaceholderRegion& orig, size_t offset); // On Windows, replaces the placeholder via VirtualAlloc2(MEM_REPLACE_PLACEHOLDER). // On POSIX, this is just a no-op. @@ -557,11 +561,12 @@ class os: AllStatic { // If addr is non-null, attempts to place the reservation at that address. static PlaceholderRegion reserve_placeholder_memory(size_t bytes, MemTag mem_tag, bool executable = false, char* addr = nullptr); - // Split 'region' at 'offset'. Returns the leading piece [base, base+offset), - // shrinks 'region' to the trailing piece [base+offset, base+original_size). + // Split 'orig' at 'offset'. Returns leading and trailing placeholder pieces as a PlaceholderRegionPair. + // The caller must not use 'orig' afterward; only 'left' and 'right' describe the OS state. // Offset must be page-aligned. - // If offset == region.size(), returns the entire region and sets region to empty. - static PlaceholderRegion split_memory(PlaceholderRegion& region, size_t offset); + // If offset == orig.size(), returns { orig, empty }. + // On failure, returns {empty, empty}. + static PlaceholderRegionPair split_memory(const PlaceholderRegion& orig, size_t offset); // Convert a placeholder region into a regular reserved region. // After conversion the Placeholder region should no longer be used. diff --git a/test/hotspot/gtest/runtime/test_os.cpp b/test/hotspot/gtest/runtime/test_os.cpp index f4e97c13cb2..fa34bf3f4fb 100644 --- a/test/hotspot/gtest/runtime/test_os.cpp +++ b/test/hotspot/gtest/runtime/test_os.cpp @@ -1265,19 +1265,19 @@ TEST_VM(os, splittable_split_two_way) { ASSERT_FALSE(region.is_empty()); char* original_base = region.base(); - os::PlaceholderRegion leading = os::split_memory(region, split_offset); + os::PlaceholderRegionPair split = os::split_memory(region, split_offset); // Leading piece: [base, base+split_offset) - ASSERT_EQ(leading.base(), original_base); - ASSERT_EQ(leading.size(), split_offset); + ASSERT_EQ(split.left.base(), original_base); + ASSERT_EQ(split.left.size(), split_offset); - // Trailing piece (region): [base+split_offset, base+total) - ASSERT_EQ(region.base(), original_base + split_offset); - ASSERT_EQ(region.size(), total - split_offset); + // Trailing piece: [base+split_offset, base+total) + ASSERT_EQ(split.right.base(), original_base + split_offset); + ASSERT_EQ(split.right.size(), total - split_offset); // Convert both and commit. - char* addr1 = os::convert_to_reserved(leading); - char* addr2 = os::convert_to_reserved(region); + char* addr1 = os::convert_to_reserved(split.left); + char* addr2 = os::convert_to_reserved(split.right); ASSERT_EQ(addr1, original_base); ASSERT_EQ(addr2, original_base + split_offset);