From 347d5728e69ae1f7d1a24820cc2c17bb0b8c0af5 Mon Sep 17 00:00:00 2001
From: Stefan Johansson <sjohanss@openjdk.org>
Date: Mon, 9 Sep 2024 11:14:26 +0000
Subject: [PATCH] 8339387: ZGC: Synchronize medium page allocation

Reviewed-by: aboldtch, stefank, eosterlund
---
 src/hotspot/share/gc/z/zObjectAllocator.cpp | 42 ++++++++++++++++++++-
 src/hotspot/share/gc/z/zObjectAllocator.hpp |  7 +++-
 2 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/src/hotspot/share/gc/z/zObjectAllocator.cpp b/src/hotspot/share/gc/z/zObjectAllocator.cpp
index 27a5965700f..bf6dc98fc72 100644
--- a/src/hotspot/share/gc/z/zObjectAllocator.cpp
+++ b/src/hotspot/share/gc/z/zObjectAllocator.cpp
@@ -25,6 +25,7 @@
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zHeap.inline.hpp"
 #include "gc/z/zHeuristics.hpp"
+#include "gc/z/zLock.inline.hpp"
 #include "gc/z/zObjectAllocator.hpp"
 #include "gc/z/zPage.inline.hpp"
 #include "gc/z/zPageTable.inline.hpp"
@@ -45,8 +46,9 @@ ZObjectAllocator::ZObjectAllocator(ZPageAge age)
     _use_per_cpu_shared_small_pages(ZHeuristics::use_per_cpu_shared_small_pages()),
     _used(0),
     _undone(0),
+    _shared_small_page(nullptr),
     _shared_medium_page(nullptr),
-    _shared_small_page(nullptr) {}
+    _medium_page_alloc_lock() {}
 
 ZPage** ZObjectAllocator::shared_small_page_addr() {
   return _use_per_cpu_shared_small_pages ? _shared_small_page.addr() : _shared_small_page.addr(0);
@@ -126,6 +128,42 @@ zaddress ZObjectAllocator::alloc_object_in_shared_page(ZPage** shared_page,
   return addr;
 }
 
+zaddress ZObjectAllocator::alloc_object_in_medium_page(size_t size,
+                                                       ZAllocationFlags flags) {
+  zaddress addr = zaddress::null;
+  ZPage** shared_medium_page = _shared_medium_page.addr();
+  ZPage* page = Atomic::load_acquire(shared_medium_page);
+
+  if (page != nullptr) {
+    addr = page->alloc_object_atomic(size);
+  }
+
+  if (is_null(addr)) {
+    // When a new medium page is required, we synchronize the allocation
+    // of the new page using a lock. This is to avoid having multiple
+    // threads requesting a medium page from the page cache when we know
+    // only one of the will succeed in installing the page at this layer.
+    ZLocker<ZLock> locker(&_medium_page_alloc_lock);
+
+    // When holding the lock we can't allow the page allocator to stall,
+    // which in the common case it won't. The page allocation is thus done
+    // in a non-blocking fashion and only if this fails we below (while not
+    // holding the lock) do the blocking page allocation.
+    ZAllocationFlags non_blocking_flags = flags;
+    non_blocking_flags.set_non_blocking();
+
+    addr = alloc_object_in_shared_page(shared_medium_page, ZPageType::medium, ZPageSizeMedium, size, non_blocking_flags);
+  }
+
+  if (is_null(addr) && !flags.non_blocking()) {
+    // The above allocation attempts failed and this allocation should stall
+    // until memory is available. Redo the allocation with blocking enabled.
+    addr = alloc_object_in_shared_page(shared_medium_page, ZPageType::medium, ZPageSizeMedium, size, flags);
+  }
+
+  return addr;
+}
+
 zaddress ZObjectAllocator::alloc_large_object(size_t size, ZAllocationFlags flags) {
   zaddress addr = zaddress::null;
 
@@ -141,7 +179,7 @@ zaddress ZObjectAllocator::alloc_large_object(size_t size, ZAllocationFlags flag
 }
 
 zaddress ZObjectAllocator::alloc_medium_object(size_t size, ZAllocationFlags flags) {
-  return alloc_object_in_shared_page(_shared_medium_page.addr(), ZPageType::medium, ZPageSizeMedium, size, flags);
+  return alloc_object_in_medium_page(size, flags);
 }
 
 zaddress ZObjectAllocator::alloc_small_object(size_t size, ZAllocationFlags flags) {
diff --git a/src/hotspot/share/gc/z/zObjectAllocator.hpp b/src/hotspot/share/gc/z/zObjectAllocator.hpp
index 8aa185646fa..5377972b7ba 100644
--- a/src/hotspot/share/gc/z/zObjectAllocator.hpp
+++ b/src/hotspot/share/gc/z/zObjectAllocator.hpp
@@ -26,6 +26,7 @@
 
 #include "gc/z/zAddress.hpp"
 #include "gc/z/zAllocationFlags.hpp"
+#include "gc/z/zLock.hpp"
 #include "gc/z/zPageAge.hpp"
 #include "gc/z/zPageType.hpp"
 #include "gc/z/zValue.hpp"
@@ -39,8 +40,9 @@ private:
   const bool         _use_per_cpu_shared_small_pages;
   ZPerCPU<size_t>    _used;
   ZPerCPU<size_t>    _undone;
-  ZContended<ZPage*> _shared_medium_page;
   ZPerCPU<ZPage*>    _shared_small_page;
+  ZContended<ZPage*> _shared_medium_page;
+  ZLock              _medium_page_alloc_lock;
 
   ZPage** shared_small_page_addr();
   ZPage* const* shared_small_page_addr() const;
@@ -56,6 +58,9 @@ private:
                                        size_t size,
                                        ZAllocationFlags flags);
 
+  zaddress alloc_object_in_medium_page(size_t size,
+                                       ZAllocationFlags flags);
+
   zaddress alloc_large_object(size_t size, ZAllocationFlags flags);
   zaddress alloc_medium_object(size_t size, ZAllocationFlags flags);
   zaddress alloc_small_object(size_t size, ZAllocationFlags flags);