infernap12/jdk
1
0
Fork 0
mirror of https://github.com/openjdk/jdk.git synced 2026-01-28 03:58:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

8374678: ZGC: Convert zForwarding to use Atomic<T>

Browse Source 
Reviewed-by: stefank, eosterlund
This commit is contained in:
Axel Boldt-Christmas 2026-01-26 13:53:12 +00:00
parent 319e21e9b4
commit 512f95cf26
4 changed files with 37 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/hotspot/share/gc/z/vmStructs_z.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -111,7 +111,6 @@ typedef ZValue<ZPerNUMAStorage, ZPartition> ZPerNUMAZPartition;
\
nonstatic_field(ZForwarding, _virtual, const ZVirtualMemory) \
nonstatic_field(ZForwarding, _object_alignment_shift, const size_t) \
volatile_nonstatic_field(ZForwarding, _ref_count, int) \
nonstatic_field(ZForwarding, _entries, const ZAttachedArrayForForwarding) \
nonstatic_field(ZForwardingEntry, _entry, uint64_t) \
nonstatic_field(ZAttachedArrayForForwarding, _length, const size_t)

45
src/hotspot/share/gc/z/zForwarding.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -29,7 +29,6 @@
#include "gc/z/zStat.hpp"
#include "gc/z/zUtils.inline.hpp"
#include "logging/log.hpp"
#include "runtime/atomicAccess.hpp"
#include "utilities/align.hpp"
//
@ -50,7 +49,7 @@
//
bool ZForwarding::claim() {
return AtomicAccess::cmpxchg(&_claimed, false, true) == false;
return _claimed.compare_set(false, true);
}
void ZForwarding::in_place_relocation_start(zoffset relocated_watermark) {
@ -60,7 +59,7 @@ void ZForwarding::in_place_relocation_start(zoffset relocated_watermark) {
// Support for ZHeap::is_in checks of from-space objects
// in a page that is in-place relocating
AtomicAccess::store(&_in_place_thread, Thread::current());
_in_place_thread.store_relaxed(Thread::current());
_in_place_top_at_start = _page->top();
}
@ -76,17 +75,17 @@ void ZForwarding::in_place_relocation_finish() {
}
// Disable relaxed ZHeap::is_in checks
AtomicAccess::store(&_in_place_thread, (Thread*)nullptr);
_in_place_thread.store_relaxed(nullptr);
}
bool ZForwarding::in_place_relocation_is_below_top_at_start(zoffset offset) const {
// Only the relocating thread is allowed to know about the old relocation top.
return AtomicAccess::load(&_in_place_thread) == Thread::current() && offset < _in_place_top_at_start;
return _in_place_thread.load_relaxed() == Thread::current() && offset < _in_place_top_at_start;
}
bool ZForwarding::retain_page(ZRelocateQueue* queue) {
for (;;) {
const int32_t ref_count = AtomicAccess::load_acquire(&_ref_count);
const int32_t ref_count = _ref_count.load_acquire();
if (ref_count == 0) {
// Released
@ -101,7 +100,7 @@ bool ZForwarding::retain_page(ZRelocateQueue* queue) {
return false;
}
if (AtomicAccess::cmpxchg(&_ref_count, ref_count, ref_count + 1) == ref_count) {
if (_ref_count.compare_set(ref_count, ref_count + 1)) {
// Retained
return true;
}
@ -110,11 +109,11 @@ bool ZForwarding::retain_page(ZRelocateQueue* queue) {
void ZForwarding::in_place_relocation_claim_page() {
for (;;) {
const int32_t ref_count = AtomicAccess::load(&_ref_count);
const int32_t ref_count = _ref_count.load_relaxed();
assert(ref_count > 0, "Invalid state");
// Invert reference count
if (AtomicAccess::cmpxchg(&_ref_count, ref_count, -ref_count) != ref_count) {
if (!_ref_count.compare_set(ref_count, -ref_count)) {
continue;
}
@ -122,7 +121,7 @@ void ZForwarding::in_place_relocation_claim_page() {
// and we have now claimed the page. Otherwise we wait until it is claimed.
if (ref_count != 1) {
ZLocker<ZConditionLock> locker(&_ref_lock);
while (AtomicAccess::load_acquire(&_ref_count) != -1) {
while (_ref_count.load_acquire() != -1) {
_ref_lock.wait();
}
}
@ -134,12 +133,12 @@ void ZForwarding::in_place_relocation_claim_page() {
void ZForwarding::release_page() {
for (;;) {
const int32_t ref_count = AtomicAccess::load(&_ref_count);
const int32_t ref_count = _ref_count.load_relaxed();
assert(ref_count != 0, "Invalid state");
if (ref_count > 0) {
// Decrement reference count
if (AtomicAccess::cmpxchg(&_ref_count, ref_count, ref_count - 1) != ref_count) {
if (!_ref_count.compare_set(ref_count, ref_count - 1)) {
continue;
}
@ -152,7 +151,7 @@ void ZForwarding::release_page() {
}
} else {
// Increment reference count
if (AtomicAccess::cmpxchg(&_ref_count, ref_count, ref_count + 1) != ref_count) {
if (!_ref_count.compare_set(ref_count, ref_count + 1)) {
continue;
}
@ -171,9 +170,9 @@ void ZForwarding::release_page() {
ZPage* ZForwarding::detach_page() {
// Wait until released
if (AtomicAccess::load_acquire(&_ref_count) != 0) {
if (_ref_count.load_acquire() != 0) {
ZLocker<ZConditionLock> locker(&_ref_lock);
while (AtomicAccess::load_acquire(&_ref_count) != 0) {
while (_ref_count.load_acquire() != 0) {
_ref_lock.wait();
}
}
@ -182,16 +181,16 @@ ZPage* ZForwarding::detach_page() {
}
ZPage* ZForwarding::page() {
assert(AtomicAccess::load(&_ref_count) != 0, "The page has been released/detached");
assert(_ref_count.load_relaxed() != 0, "The page has been released/detached");
return _page;
}
void ZForwarding::mark_done() {
AtomicAccess::store(&_done, true);
_done.store_relaxed(true);
}
bool ZForwarding::is_done() const {
return AtomicAccess::load(&_done);
return _done.load_relaxed();
}
//
@ -288,7 +287,7 @@ void ZForwarding::relocated_remembered_fields_publish() {
// used to have remembered set entries. Now publish the fields to
// the YC.
const ZPublishState res = AtomicAccess::cmpxchg(&_relocated_remembered_fields_state, ZPublishState::none, ZPublishState::published);
const ZPublishState res = _relocated_remembered_fields_state.compare_exchange(ZPublishState::none, ZPublishState::published);
// none: OK to publish
// published: Not possible - this operation makes this transition
@ -319,7 +318,7 @@ void ZForwarding::relocated_remembered_fields_notify_concurrent_scan_of() {
// Invariant: The page is being retained
assert(ZGeneration::young()->is_phase_mark(), "Only called when");
const ZPublishState res = AtomicAccess::cmpxchg(&_relocated_remembered_fields_state, ZPublishState::none, ZPublishState::reject);
const ZPublishState res = _relocated_remembered_fields_state.compare_exchange(ZPublishState::none, ZPublishState::reject);
// none: OC has not completed relocation
// published: OC has completed and published all relocated remembered fields
@ -340,7 +339,7 @@ void ZForwarding::relocated_remembered_fields_notify_concurrent_scan_of() {
// OC relocation already collected and published fields
// Still notify concurrent scanning and reject the collected data from the OC
const ZPublishState res2 = AtomicAccess::cmpxchg(&_relocated_remembered_fields_state, ZPublishState::published, ZPublishState::reject);
const ZPublishState res2 = _relocated_remembered_fields_state.compare_exchange(ZPublishState::published, ZPublishState::reject);
assert(res2 == ZPublishState::published, "Should not fail");
log_debug(gc, remset)("Forwarding remset eager and reject: " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end()));
@ -368,7 +367,7 @@ bool ZForwarding::relocated_remembered_fields_published_contains(volatile zpoint
}
void ZForwarding::verify() const {
guarantee(_ref_count != 0, "Invalid reference count");
guarantee(_ref_count.load_relaxed() != 0, "Invalid reference count");
guarantee(_page != nullptr, "Invalid page");
uint32_t live_objects = 0;

13
src/hotspot/share/gc/z/zForwarding.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -32,6 +32,7 @@
#include "gc/z/zPageAge.hpp"
#include "gc/z/zPageType.hpp"
#include "gc/z/zVirtualMemory.hpp"
#include "runtime/atomic.hpp"
class ObjectClosure;
class ZForwardingAllocator;
@ -62,13 +63,13 @@ private:
const uint32_t _partition_id;
const ZPageAge _from_age;
const ZPageAge _to_age;
volatile bool _claimed;
Atomic<bool> _claimed;
mutable ZConditionLock _ref_lock;
volatile int32_t _ref_count;
volatile bool _done;
Atomic<int32_t> _ref_count;
Atomic<bool> _done;
// Relocated remembered set fields support
volatile ZPublishState _relocated_remembered_fields_state;
Atomic<ZPublishState> _relocated_remembered_fields_state;
PointerArray _relocated_remembered_fields_array;
uint32_t _relocated_remembered_fields_publish_young_seqnum;
@ -77,7 +78,7 @@ private:
zoffset_end _in_place_top_at_start;
// Debugging
volatile Thread* _in_place_thread;
Atomic<Thread*> _in_place_thread;
ZForwardingEntry* entries() const;
ZForwardingEntry at(ZForwardingCursor* cursor) const;

14
src/hotspot/share/gc/z/zForwarding.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -196,7 +196,7 @@ void ZForwarding::oops_do_in_forwarded_via_table(Function function) {
}
inline bool ZForwarding::in_place_relocation() const {
assert(AtomicAccess::load(&_ref_count) != 0, "The page has been released/detached");
assert(_ref_count.load_relaxed() != 0, "The page has been released/detached");
return _in_place;
}
@ -307,7 +307,7 @@ inline void ZForwarding::relocated_remembered_fields_register(volatile zpointer*
// Invariant: Page is being retained
assert(ZGeneration::young()->is_phase_mark(), "Only called when");
const ZPublishState res = AtomicAccess::load(&_relocated_remembered_fields_state);
const ZPublishState res = _relocated_remembered_fields_state.load_relaxed();
// none: Gather remembered fields
// published: Have already published fields - not possible since they haven't been
@ -327,7 +327,7 @@ inline void ZForwarding::relocated_remembered_fields_register(volatile zpointer*
// Returns true iff the page is being (or about to be) relocated by the OC
// while the YC gathered the remembered fields of the "from" page.
inline bool ZForwarding::relocated_remembered_fields_is_concurrently_scanned() const {
return AtomicAccess::load(&_relocated_remembered_fields_state) == ZPublishState::reject;
return _relocated_remembered_fields_state.load_relaxed() == ZPublishState::reject;
}
template <typename Function>
@ -335,7 +335,7 @@ inline void ZForwarding::relocated_remembered_fields_apply_to_published(Function
// Invariant: Page is not being retained
assert(ZGeneration::young()->is_phase_mark(), "Only called when");
const ZPublishState res = AtomicAccess::load_acquire(&_relocated_remembered_fields_state);
const ZPublishState res = _relocated_remembered_fields_state.load_acquire();
// none: Nothing published - page had already been relocated before YC started
// published: OC relocated and published relocated remembered fields
@ -363,14 +363,14 @@ inline void ZForwarding::relocated_remembered_fields_apply_to_published(Function
// collection. Mark that it is unsafe (and unnecessary) to call scan_page
// on the page in the page table.
assert(res != ZPublishState::accept, "Unexpected");
AtomicAccess::store(&_relocated_remembered_fields_state, ZPublishState::reject);
_relocated_remembered_fields_state.store_relaxed(ZPublishState::reject);
} else {
log_debug(gc, remset)("scan_forwarding failed retain safe " PTR_FORMAT, untype(start()));
// Guaranteed that the page was fully relocated and removed from page table.
// Because of this we can signal to scan_page that any page found in page table
// of the same slot as the current forwarding is a page that is safe to scan,
// and in fact must be scanned.
AtomicAccess::store(&_relocated_remembered_fields_state, ZPublishState::accept);
_relocated_remembered_fields_state.store_relaxed(ZPublishState::accept);
}
}