infernap12/jdk
1
0
Fork 0
mirror of https://github.com/openjdk/jdk.git synced 2026-03-15 18:33:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

8242088: Replace mutually exclusive lists with concurrent alternatives

Browse Source 
Reviewed-by: egahlin
This commit is contained in:
Markus Grönlund 2020-06-05 14:59:27 +02:00
parent 4de4200652
commit 45fa5aa699
37 changed files with 2318 additions and 1156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/hotspot/share/jfr/jfr.cpp
View File

@ -25,7 +25,6 @@
#include "precompiled.hpp"
#include "jfr/jfr.hpp"
#include "jfr/leakprofiler/leakProfiler.hpp"
#include "jfr/periodic/sampling/jfrThreadSampler.hpp"
#include "jfr/recorder/jfrRecorder.hpp"
#include "jfr/recorder/checkpoint/jfrCheckpointManager.hpp"
#include "jfr/recorder/repository/jfrEmergencyDump.hpp"
@ -33,6 +32,7 @@
#include "jfr/recorder/repository/jfrRepository.hpp"
#include "jfr/support/jfrThreadLocal.hpp"
#include "runtime/java.hpp"
#include "runtime/thread.hpp"
bool Jfr::is_enabled() {
return JfrRecorder::is_enabled();

202
src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp
View File

@ -38,6 +38,7 @@
#include "jfr/recorder/storage/jfrStorageUtils.inline.hpp"
#include "jfr/utilities/jfrBigEndian.hpp"
#include "jfr/utilities/jfrIterator.hpp"
#include "jfr/utilities/jfrLinkedList.inline.hpp"
#include "jfr/utilities/jfrThreadIterator.hpp"
#include "jfr/utilities/jfrTypes.hpp"
#include "jfr/writers/jfrJavaEventWriter.hpp"
@ -50,9 +51,7 @@
#include "runtime/os.inline.hpp"
#include "runtime/safepoint.hpp"
typedef JfrCheckpointManager::Buffer* BufferPtr;
static JfrCheckpointManager* _instance = NULL;
typedef JfrCheckpointManager::BufferPtr BufferPtr;
static volatile bool constant_pending = false;
@ -70,6 +69,8 @@ static void set_constant_pending() {
}
}
static JfrCheckpointManager* _instance = NULL;
JfrCheckpointManager& JfrCheckpointManager::instance() {
return *_instance;
}
@ -89,7 +90,6 @@ void JfrCheckpointManager::destroy() {
JfrCheckpointManager::JfrCheckpointManager(JfrChunkWriter& cw) :
_free_list_mspace(NULL),
_epoch_transition_mspace(NULL),
_lock(NULL),
_service_thread(NULL),
_chunkwriter(cw),
_checkpoint_epoch_state(JfrTraceIdEpoch::epoch()) {}
@ -101,9 +101,6 @@ JfrCheckpointManager::~JfrCheckpointManager() {
if (_epoch_transition_mspace != NULL) {
delete _epoch_transition_mspace;
}
if (_lock != NULL) {
delete _lock;
}
JfrTypeManager::destroy();
}
@ -126,11 +123,6 @@ bool JfrCheckpointManager::initialize() {
if (_epoch_transition_mspace == NULL) {
return false;
}
assert(_lock == NULL, "invariant");
_lock = new Mutex(Monitor::leaf - 1, "Checkpoint mutex", Mutex::_allow_vm_block_flag, Monitor::_safepoint_check_never);
if (_lock == NULL) {
return false;
}
return JfrTypeManager::initialize();
}
@ -138,28 +130,15 @@ void JfrCheckpointManager::register_service_thread(const Thread* thread) {
_service_thread = thread;
}
void JfrCheckpointManager::register_full(BufferPtr t, Thread* thread) {
void JfrCheckpointManager::register_full(BufferPtr buffer, Thread* thread) {
// nothing here at the moment
assert(t != NULL, "invariant");
assert(t->acquired_by(thread), "invariant");
assert(t->retired(), "invariant");
}
void JfrCheckpointManager::lock() {
assert(!_lock->owned_by_self(), "invariant");
_lock->lock_without_safepoint_check();
}
void JfrCheckpointManager::unlock() {
_lock->unlock();
assert(buffer != NULL, "invariant");
assert(buffer->acquired_by(thread), "invariant");
assert(buffer->retired(), "invariant");
}
#ifdef ASSERT
bool JfrCheckpointManager::is_locked() const {
return _lock->owned_by_self();
}
static void assert_free_lease(const BufferPtr buffer) {
static void assert_lease(const BufferPtr buffer) {
assert(buffer != NULL, "invariant");
assert(buffer->acquired_by_self(), "invariant");
assert(buffer->lease(), "invariant");
@ -172,45 +151,46 @@ static void assert_release(const BufferPtr buffer) {
}
#endif // ASSERT
static BufferPtr lease_free(size_t size, JfrCheckpointMspace* mspace, size_t retry_count, Thread* thread) {
static const size_t max_elem_size = mspace->min_elem_size(); // min is max
BufferPtr buffer;
if (size <= max_elem_size) {
BufferPtr buffer = mspace_get_free_lease_with_retry(size, mspace, retry_count, thread);
if (buffer != NULL) {
DEBUG_ONLY(assert_free_lease(buffer);)
return buffer;
}
}
buffer = mspace_allocate_transient_lease_to_free(size, mspace, thread);
DEBUG_ONLY(assert_free_lease(buffer);)
return buffer;
}
bool JfrCheckpointManager::use_epoch_transition_mspace(const Thread* thread) const {
return _service_thread != thread && Atomic::load_acquire(&_checkpoint_epoch_state) != JfrTraceIdEpoch::epoch();
}
static const size_t lease_retry = 10;
BufferPtr JfrCheckpointManager::lease_buffer(Thread* thread, size_t size /* 0 */) {
JfrCheckpointManager& manager = instance();
if (manager.use_epoch_transition_mspace(thread)) {
return lease_free(size, manager._epoch_transition_mspace, lease_retry, thread);
BufferPtr JfrCheckpointManager::lease(JfrCheckpointMspace* mspace, Thread* thread, size_t size /* 0 */) {
assert(mspace != NULL, "invariant");
static const size_t max_elem_size = mspace->min_elem_size(); // min is max
BufferPtr buffer;
if (size <= max_elem_size) {
buffer = mspace_get_free_lease_with_retry(size, mspace, lease_retry, thread);
if (buffer != NULL) {
DEBUG_ONLY(assert_lease(buffer);)
return buffer;
}
}
return lease_free(size, manager._free_list_mspace, lease_retry, thread);
buffer = mspace_allocate_transient_lease_to_full(size, mspace, thread);
DEBUG_ONLY(assert_lease(buffer);)
return buffer;
}
BufferPtr JfrCheckpointManager::lease(Thread* thread, size_t size /* 0 */) {
JfrCheckpointManager& manager = instance();
JfrCheckpointMspace* const mspace = manager.use_epoch_transition_mspace(thread) ?
manager._epoch_transition_mspace :
manager._free_list_mspace;
return lease(mspace, thread, size);
}
JfrCheckpointMspace* JfrCheckpointManager::lookup(BufferPtr old) const {
assert(old != NULL, "invariant");
return _free_list_mspace->in_free_list(old) ? _free_list_mspace : _epoch_transition_mspace;
return _free_list_mspace->in_mspace(old) ? _free_list_mspace : _epoch_transition_mspace;
}
BufferPtr JfrCheckpointManager::lease_buffer(BufferPtr old, Thread* thread, size_t size /* 0 */) {
BufferPtr JfrCheckpointManager::lease(BufferPtr old, Thread* thread, size_t size /* 0 */) {
assert(old != NULL, "invariant");
JfrCheckpointMspace* mspace = instance().lookup(old);
assert(mspace != NULL, "invariant");
return lease_free(size, mspace, lease_retry, thread);
return lease(mspace, thread, size);
}
/*
@ -219,10 +199,14 @@ BufferPtr JfrCheckpointManager::lease_buffer(BufferPtr old, Thread* thread, size
* The buffer is effectively invalidated for the thread post-return,
* and the caller should take means to ensure that it is not referenced.
*/
static void release(BufferPtr const buffer, Thread* thread) {
static void release(BufferPtr buffer, Thread* thread) {
DEBUG_ONLY(assert_release(buffer);)
buffer->clear_lease();
buffer->release();
if (buffer->transient()) {
buffer->set_retired();
} else {
buffer->release();
}
}
BufferPtr JfrCheckpointManager::flush(BufferPtr old, size_t used, size_t requested, Thread* thread) {
@ -235,7 +219,7 @@ BufferPtr JfrCheckpointManager::flush(BufferPtr old, size_t used, size_t request
return NULL;
}
// migration of in-flight information
BufferPtr const new_buffer = lease_buffer(old, thread, used + requested);
BufferPtr const new_buffer = lease(old, thread, used + requested);
if (new_buffer != NULL) {
migrate_outstanding_writes(old, new_buffer, used, requested);
}
@ -335,18 +319,22 @@ class CheckpointWriteOp {
size_t processed() const { return _processed; }
};
typedef CheckpointWriteOp<JfrCheckpointMspace::Type> WriteOperation;
typedef ReleaseOp<JfrCheckpointMspace> CheckpointReleaseOperation;
typedef CheckpointWriteOp<JfrCheckpointManager::Buffer> WriteOperation;
typedef ReleaseOp<JfrCheckpointMspace> CheckpointReleaseFreeOperation;
typedef ScavengingReleaseOp<JfrCheckpointMspace> CheckpointReleaseFullOperation;
template <template <typename> class WriterHost, template <typename, typename, typename> class CompositeOperation>
template <template <typename> class WriterHost>
static size_t write_mspace(JfrCheckpointMspace* mspace, JfrChunkWriter& chunkwriter) {
assert(mspace != NULL, "invariant");
WriteOperation wo(chunkwriter);
WriterHost<WriteOperation> wh(wo);
CheckpointReleaseOperation cro(mspace, Thread::current(), false);
CompositeOperation<WriterHost<WriteOperation>, CheckpointReleaseOperation, CompositeOperationAnd> co(&wh, &cro);
assert(mspace->is_full_empty(), "invariant");
process_free_list(co, mspace);
CheckpointReleaseFreeOperation free_release_op(mspace);
CompositeOperation<WriterHost<WriteOperation>, CheckpointReleaseFreeOperation> free_op(&wh, &free_release_op);
process_free_list(free_op, mspace);
CheckpointReleaseFullOperation full_release_op(mspace);
MutexedWriteOp<WriteOperation> full_write_op(wo);
CompositeOperation<MutexedWriteOp<WriteOperation>, CheckpointReleaseFullOperation> full_op(&full_write_op, &full_release_op);
process_full_list(full_op, mspace);
return wo.processed();
}
@ -369,52 +357,66 @@ void JfrCheckpointManager::synchronize_checkpoint_manager_with_current_epoch() {
}
size_t JfrCheckpointManager::write() {
const size_t processed = write_mspace<MutexedWriteOp, CompositeOperation>(_free_list_mspace, _chunkwriter);
const size_t processed = write_mspace<MutexedWriteOp>(_free_list_mspace, _chunkwriter);
synchronize_checkpoint_manager_with_current_epoch();
return processed;
}
size_t JfrCheckpointManager::write_epoch_transition_mspace() {
return write_mspace<ExclusiveOp, CompositeOperation>(_epoch_transition_mspace, _chunkwriter);
return write_mspace<ExclusiveOp>(_epoch_transition_mspace, _chunkwriter);
}
typedef DiscardOp<DefaultDiscarder<JfrBuffer> > DiscardOperation;
typedef DiscardOp<DefaultDiscarder<JfrCheckpointManager::Buffer> > DiscardOperation;
typedef ExclusiveDiscardOp<DefaultDiscarder<JfrCheckpointManager::Buffer> > DiscardOperationEpochTransitionMspace;
typedef CompositeOperation<DiscardOperation, CheckpointReleaseFreeOperation> DiscardFreeOperation;
typedef CompositeOperation<DiscardOperation, CheckpointReleaseFullOperation> DiscardFullOperation;
typedef CompositeOperation<DiscardOperationEpochTransitionMspace, CheckpointReleaseFreeOperation> DiscardEpochTransMspaceFreeOperation;
typedef CompositeOperation<DiscardOperationEpochTransitionMspace, CheckpointReleaseFullOperation> DiscardEpochTransMspaceFullOperation;
size_t JfrCheckpointManager::clear() {
clear_type_set();
DiscardOperation discarder(mutexed); // mutexed discard mode
process_free_list(discarder, _free_list_mspace);
process_free_list(discarder, _epoch_transition_mspace);
DiscardOperation mutex_discarder(mutexed);
CheckpointReleaseFreeOperation free_release_op(_free_list_mspace);
DiscardFreeOperation free_op(&mutex_discarder, &free_release_op);
process_free_list(free_op, _free_list_mspace);
CheckpointReleaseFullOperation full_release_op(_free_list_mspace);
DiscardFullOperation full_op(&mutex_discarder, &full_release_op);
process_full_list(full_op, _free_list_mspace);
DiscardOperationEpochTransitionMspace epoch_transition_discarder(mutexed);
CheckpointReleaseFreeOperation epoch_free_release_op(_epoch_transition_mspace);
DiscardEpochTransMspaceFreeOperation epoch_free_op(&epoch_transition_discarder, &epoch_free_release_op);
process_free_list(epoch_free_op, _epoch_transition_mspace);
CheckpointReleaseFullOperation epoch_full_release_op(_epoch_transition_mspace);
DiscardEpochTransMspaceFullOperation epoch_full_op(&epoch_transition_discarder, &epoch_full_release_op);
process_full_list(epoch_full_op, _epoch_transition_mspace);
synchronize_checkpoint_manager_with_current_epoch();
return discarder.elements();
return mutex_discarder.elements() + epoch_transition_discarder.elements();
}
// Optimization for write_static_type_set() and write_threads() is to write
// directly into the epoch transition mspace because we will immediately
// serialize and reset this mspace post-write.
static JfrBuffer* get_epoch_transition_buffer(JfrCheckpointMspace* mspace, Thread* t) {
assert(mspace != NULL, "invariant");
JfrBuffer* const buffer = mspace->free_head();
assert(buffer != NULL, "invariant");
buffer->acquire(t);
buffer->set_lease();
DEBUG_ONLY(assert_free_lease(buffer);)
BufferPtr JfrCheckpointManager::epoch_transition_buffer(Thread* thread) {
assert(_epoch_transition_mspace->free_list_is_nonempty(), "invariant");
BufferPtr const buffer = lease(_epoch_transition_mspace, thread, _epoch_transition_mspace->min_elem_size());
DEBUG_ONLY(assert_lease(buffer);)
return buffer;
}
size_t JfrCheckpointManager::write_static_type_set() {
Thread* const t = Thread::current();
ResourceMark rm(t);
HandleMark hm(t);
JfrCheckpointWriter writer(t, get_epoch_transition_buffer(_epoch_transition_mspace, t), STATICS);
Thread* const thread = Thread::current();
ResourceMark rm(thread);
HandleMark hm(thread);
JfrCheckpointWriter writer(thread, epoch_transition_buffer(thread), STATICS);
JfrTypeManager::write_static_types(writer);
return writer.used_size();
}
size_t JfrCheckpointManager::write_threads() {
Thread* const t = Thread::current();
ResourceMark rm(t);
HandleMark hm(t);
JfrCheckpointWriter writer(t, get_epoch_transition_buffer(_epoch_transition_mspace, t), THREADS);
Thread* const thread = Thread::current();
ResourceMark rm(thread);
HandleMark hm(thread);
JfrCheckpointWriter writer(thread, epoch_transition_buffer(thread), THREADS);
JfrTypeManager::write_threads(writer);
return writer.used_size();
}
@ -442,20 +444,20 @@ void JfrCheckpointManager::clear_type_set() {
void JfrCheckpointManager::write_type_set() {
assert(!SafepointSynchronize::is_at_safepoint(), "invariant");
Thread* const thread = Thread::current();
if (LeakProfiler::is_running()) {
Thread* const t = Thread::current();
// can safepoint here
MutexLocker cld_lock(t, ClassLoaderDataGraph_lock);
MutexLocker module_lock(t, Module_lock);
JfrCheckpointWriter leakp_writer(t);
JfrCheckpointWriter writer(t);
MutexLocker cld_lock(thread, ClassLoaderDataGraph_lock);
MutexLocker module_lock(thread, Module_lock);
JfrCheckpointWriter leakp_writer(thread);
JfrCheckpointWriter writer(thread);
JfrTypeSet::serialize(&writer, &leakp_writer, false, false);
ObjectSampleCheckpoint::on_type_set(leakp_writer);
} else {
// can safepoint here
MutexLocker cld_lock(ClassLoaderDataGraph_lock);
MutexLocker module_lock(Module_lock);
JfrCheckpointWriter writer(Thread::current());
JfrCheckpointWriter writer(thread);
JfrTypeSet::serialize(&writer, NULL, false, false);
}
write();
@ -489,27 +491,27 @@ size_t JfrCheckpointManager::flush_type_set() {
if (is_constant_pending()) {
WriteOperation wo(_chunkwriter);
FlushOperation fo(wo);
assert(_free_list_mspace->is_full_empty(), "invariant");
process_free_list(fo, _free_list_mspace);
process_full_list(fo, _free_list_mspace);
}
return elements;
}
void JfrCheckpointManager::create_thread_blob(Thread* t) {
JfrTypeManager::create_thread_blob(t);
void JfrCheckpointManager::create_thread_blob(Thread* thread) {
JfrTypeManager::create_thread_blob(thread);
}
void JfrCheckpointManager::write_thread_checkpoint(Thread* t) {
JfrTypeManager::write_thread_checkpoint(t);
void JfrCheckpointManager::write_thread_checkpoint(Thread* thread) {
JfrTypeManager::write_thread_checkpoint(thread);
}
class JfrNotifyClosure : public ThreadClosure {
public:
void do_thread(Thread* t) {
assert(t != NULL, "invariant");
assert(t->is_Java_thread(), "invariant");
void do_thread(Thread* thread) {
assert(thread != NULL, "invariant");
assert(thread->is_Java_thread(), "invariant");
assert_locked_or_safepoint(Threads_lock);
JfrJavaEventWriter::notify((JavaThread*)t);
JfrJavaEventWriter::notify((JavaThread*)thread);
}
};

41
src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -28,6 +28,7 @@
#include "jfr/recorder/storage/jfrBuffer.hpp"
#include "jfr/recorder/storage/jfrMemorySpace.hpp"
#include "jfr/recorder/storage/jfrMemorySpaceRetrieval.hpp"
#include "jfr/utilities/jfrLinkedList.hpp"
class JfrCheckpointManager;
class JfrChunkWriter;
@ -44,7 +45,7 @@ struct JfrCheckpointEntry {
juint nof_segments;
};
typedef JfrMemorySpace<JfrBuffer, JfrMspaceSequentialRetrieval, JfrCheckpointManager> JfrCheckpointMspace;
typedef JfrMemorySpace<JfrCheckpointManager, JfrMspaceRetrieval, JfrLinkedList<JfrBuffer> > JfrCheckpointMspace;
//
// Responsible for maintaining checkpoints and by implication types.
@ -53,28 +54,29 @@ typedef JfrMemorySpace<JfrBuffer, JfrMspaceSequentialRetrieval, JfrCheckpointMan
//
class JfrCheckpointManager : public JfrCHeapObj {
public:
typedef JfrCheckpointMspace::Type Buffer;
size_t flush_type_set();
static void create_thread_blob(Thread* thread);
static void write_thread_checkpoint(Thread* thread);
void register_service_thread(const Thread* thread);
typedef JfrCheckpointMspace::Node Buffer;
typedef JfrCheckpointMspace::NodePtr BufferPtr;
private:
JfrCheckpointMspace* _free_list_mspace;
JfrCheckpointMspace* _epoch_transition_mspace;
Mutex* _lock;
const Thread* _service_thread;
JfrChunkWriter& _chunkwriter;
bool _checkpoint_epoch_state;
// mspace callback
void register_full(Buffer* t, Thread* thread);
void lock();
void unlock();
DEBUG_ONLY(bool is_locked() const;)
JfrCheckpointMspace* lookup(Buffer* old) const;
bool use_epoch_transition_mspace(const Thread* t) const;
JfrCheckpointMspace* lookup(BufferPtr old) const;
bool use_epoch_transition_mspace(const Thread* thread) const;
size_t write_epoch_transition_mspace();
BufferPtr epoch_transition_buffer(Thread* thread);
static Buffer* lease_buffer(Thread* t, size_t size = 0);
static Buffer* lease_buffer(Buffer* old, Thread* t, size_t size = 0);
static Buffer* flush(Buffer* old, size_t used, size_t requested, Thread* t);
static BufferPtr lease(Thread* thread, size_t size = 0);
static BufferPtr lease(BufferPtr old, Thread* thread, size_t size = 0);
static BufferPtr lease(JfrCheckpointMspace* mspace, Thread* thread, size_t size = 0);
static BufferPtr flush(BufferPtr old, size_t used, size_t requested, Thread* thread);
size_t clear();
size_t write();
@ -102,11 +104,8 @@ class JfrCheckpointManager : public JfrCHeapObj {
void on_rotation();
static void destroy();
public:
size_t flush_type_set();
static void create_thread_blob(Thread* t);
static void write_thread_checkpoint(Thread* t);
void register_service_thread(const Thread* t);
// mspace callback
void register_full(BufferPtr buffer, Thread* thread);
friend class Jfr;
friend class JfrRecorder;
@ -115,7 +114,7 @@ class JfrCheckpointManager : public JfrCHeapObj {
friend class JfrCheckpointWriter;
friend class JfrSerializer;
friend class JfrStackTraceRepository;
template <typename, template <typename> class, typename>
template <typename, template <typename> class, typename, typename>
friend class JfrMemorySpace;
};

6
src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointWriter.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -32,7 +32,7 @@ JfrCheckpointFlush::JfrCheckpointFlush(Type* old, size_t used, size_t requested,
_result(JfrCheckpointManager::flush(old, used, requested, t)) {}
JfrCheckpointWriter::JfrCheckpointWriter(JfrCheckpointType type /* GENERIC */) :
JfrCheckpointWriterBase(JfrCheckpointManager::lease_buffer(Thread::current()), Thread::current()),
JfrCheckpointWriterBase(JfrCheckpointManager::lease(Thread::current()), Thread::current()),
_time(JfrTicks::now()),
_offset(0),
_count(0),
@ -46,7 +46,7 @@ JfrCheckpointWriter::JfrCheckpointWriter(JfrCheckpointType type /* GENERIC */) :
}
JfrCheckpointWriter::JfrCheckpointWriter(Thread* t, bool header /* true */, JfrCheckpointType type /* GENERIC */) :
JfrCheckpointWriterBase(JfrCheckpointManager::lease_buffer(t), t),
JfrCheckpointWriterBase(JfrCheckpointManager::lease(t), t),
_time(JfrTicks::now()),
_offset(0),
_count(0),

87
src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeManager.cpp
View File

@ -28,8 +28,8 @@
#include "jfr/recorder/checkpoint/types/jfrType.hpp"
#include "jfr/recorder/checkpoint/types/jfrTypeManager.hpp"
#include "jfr/recorder/jfrRecorder.hpp"
#include "jfr/utilities/jfrDoublyLinkedList.hpp"
#include "jfr/utilities/jfrIterator.hpp"
#include "jfr/utilities/jfrLinkedList.inline.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/handles.inline.hpp"
#include "runtime/safepoint.hpp"
@ -38,38 +38,20 @@
#include "utilities/exceptions.hpp"
class JfrSerializerRegistration : public JfrCHeapObj {
public:
JfrSerializerRegistration* _next; // list support
private:
JfrSerializerRegistration* _next;
JfrSerializerRegistration* _prev;
JfrSerializer* _serializer;
mutable JfrBlobHandle _cache;
JfrTypeId _id;
bool _permit_cache;
public:
JfrSerializerRegistration(JfrTypeId id, bool permit_cache, JfrSerializer* serializer) :
_next(NULL), _prev(NULL), _serializer(serializer), _cache(), _id(id), _permit_cache(permit_cache) {}
_next(NULL), _serializer(serializer), _cache(), _id(id), _permit_cache(permit_cache) {}
~JfrSerializerRegistration() {
delete _serializer;
}
JfrSerializerRegistration* next() const {
return _next;
}
void set_next(JfrSerializerRegistration* next) {
_next = next;
}
JfrSerializerRegistration* prev() const {
return _prev;
}
void set_prev(JfrSerializerRegistration* prev) {
_prev = prev;
}
JfrTypeId id() const {
return _id;
}
@ -155,34 +137,50 @@ class SerializerRegistrationGuard : public StackObj {
Semaphore SerializerRegistrationGuard::_mutex_semaphore(1);
typedef JfrDoublyLinkedList<JfrSerializerRegistration> List;
typedef StopOnNullIterator<const List> Iterator;
typedef JfrLinkedList<JfrSerializerRegistration> List;
static List types;
void JfrTypeManager::destroy() {
SerializerRegistrationGuard guard;
Iterator iter(types);
JfrSerializerRegistration* registration;
while (iter.has_next()) {
registration = types.remove(iter.next());
while (types.is_nonempty()) {
registration = types.remove();
assert(registration != NULL, "invariant");
delete registration;
}
}
void JfrTypeManager::on_rotation() {
const Iterator iter(types);
while (iter.has_next()) {
iter.next()->on_rotation();
class InvokeOnRotation {
public:
bool process(const JfrSerializerRegistration* r) {
assert(r != NULL, "invariant");
r->on_rotation();
return true;
}
};
void JfrTypeManager::on_rotation() {
InvokeOnRotation ior;
types.iterate(ior);
}
#ifdef ASSERT
static void assert_not_registered_twice(JfrTypeId id, List& list) {
const Iterator iter(list);
while (iter.has_next()) {
assert(iter.next()->id() != id, "invariant");
class Diversity {
private:
const JfrTypeId _id;
public:
Diversity(JfrTypeId id) : _id(id) {}
bool process(const JfrSerializerRegistration* r) {
assert(r != NULL, "invariant");
assert(r->id() != _id, "invariant");
return true;
}
};
static void assert_not_registered_twice(JfrTypeId id, List& list) {
Diversity d(id);
types.iterate(d);
}
#endif
@ -199,7 +197,7 @@ static bool register_static_type(JfrTypeId id, bool permit_cache, JfrSerializer*
JfrCheckpointWriter writer(STATICS);
registration->invoke(writer);
}
types.prepend(registration);
types.add(registration);
return true;
}
@ -229,11 +227,20 @@ bool JfrSerializer::register_serializer(JfrTypeId id, bool permit_cache, JfrSeri
return register_static_type(id, permit_cache, serializer);
}
class InvokeSerializer {
private:
JfrCheckpointWriter& _writer;
public:
InvokeSerializer(JfrCheckpointWriter& writer) : _writer(writer) {}
bool process(const JfrSerializerRegistration* r) {
assert(r != NULL, "invariant");
r->invoke(_writer);
return true;
}
};
void JfrTypeManager::write_static_types(JfrCheckpointWriter& writer) {
InvokeSerializer is(writer);
SerializerRegistrationGuard guard;
const Iterator iter(types);
while (iter.has_next()) {
iter.next()->invoke(writer);
}
types.iterate(is);
}

6
src/hotspot/share/jfr/recorder/service/jfrPostBox.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2013, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -40,7 +40,6 @@ enum JFR_Msg {
MSG_WAKEUP,
MSG_SHUTDOWN,
MSG_VM_ERROR,
MSG_DEADBUFFER,
MSG_FLUSHPOINT,
MSG_NO_OF_MSGS
};
@ -55,7 +54,7 @@ enum JFR_Msg {
* MSG_STOP (2) ; MSGBIT(MSG_STOP) == (1 << 0x2) == 0x4
* MSG_ROTATE (3) ; MSGBIT(MSG_ROTATE) == (1 << 0x3) == 0x8
* MSG_VM_ERROR (8) ; MSGBIT(MSG_VM_ERROR) == (1 << 0x8) == 0x100
* MSG_FLUSHPOINT (10) ; MSGBIT(MSG_FLUSHPOINT) == (1 << 0xa) == 0x400
* MSG_FLUSHPOINT (9) ; MSGBIT(MSG_FLUSHPOINT) == (1 << 0x9) == 0x200
*
* Asynchronous messages (posting thread returns immediately upon deposit):
*
@ -63,7 +62,6 @@ enum JFR_Msg {
* MSG_CHECKPOINT (5) ; MSGBIT(CHECKPOINT) == (1 << 0x5) == 0x20
* MSG_WAKEUP (6) ; MSGBIT(WAKEUP) == (1 << 0x6) == 0x40
* MSG_SHUTDOWN (7) ; MSGBIT(MSG_SHUTDOWN) == (1 << 0x7) == 0x80
* MSG_DEADBUFFER (9) ; MSGBIT(MSG_DEADBUFFER) == (1 << 0x9) == 0x200
*/
class JfrPostBox : public JfrCHeapObj {

4
src/hotspot/share/jfr/recorder/service/jfrRecorderService.cpp
View File

@ -657,10 +657,6 @@ void JfrRecorderService::process_full_buffers() {
}
}
void JfrRecorderService::scavenge() {
_storage.scavenge();
}
void JfrRecorderService::evaluate_chunk_size_for_rotation() {
JfrChunkRotation::evaluate(_chunkwriter);
}

6
src/hotspot/share/jfr/recorder/service/jfrRecorderThreadLoop.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -42,7 +42,6 @@ void recorderthread_entry(JavaThread* thread, Thread* unused) {
#define ROTATE (msgs & (MSGBIT(MSG_ROTATE)|MSGBIT(MSG_STOP)))
#define FLUSHPOINT (msgs & (MSGBIT(MSG_FLUSHPOINT)))
#define PROCESS_FULL_BUFFERS (msgs & (MSGBIT(MSG_ROTATE)|MSGBIT(MSG_STOP)|MSGBIT(MSG_FULLBUFFER)))
#define SCAVENGE (msgs & (MSGBIT(MSG_DEADBUFFER)))
JfrPostBox& post_box = JfrRecorderThread::post_box();
log_debug(jfr, system)("Recorder thread STARTED");
@ -63,9 +62,6 @@ void recorderthread_entry(JavaThread* thread, Thread* unused) {
if (PROCESS_FULL_BUFFERS) {
service.process_full_buffers();
}
if (SCAVENGE) {
service.scavenge();
}
// Check amount of data written to chunk already
// if it warrants asking for a new chunk
service.evaluate_chunk_size_for_rotation();

13
src/hotspot/share/jfr/recorder/storage/jfrBuffer.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -29,7 +29,6 @@
static const u1* const TOP_CRITICAL_SECTION = NULL;
JfrBuffer::JfrBuffer() : _next(NULL),
_prev(NULL),
_identity(NULL),
_pos(NULL),
_top(NULL),
@ -54,7 +53,6 @@ bool JfrBuffer::initialize(size_t header_size, size_t size) {
void JfrBuffer::reinitialize(bool exclusion /* false */) {
acquire_critical_section_top();
assert(!lease(), "invariant");
assert(!transient(), "invariant");
if (exclusion != excluded()) {
// update
if (exclusion) {
@ -124,6 +122,13 @@ bool JfrBuffer::try_acquire(const void* id) {
return current_id == NULL && Atomic::cmpxchg(&_identity, current_id, id) == current_id;
}
void JfrBuffer::set_identity(const void* id) {
assert(id != NULL, "invariant");
assert(_identity == NULL, "invariant");
OrderAccess::storestore();
_identity = id;
}
void JfrBuffer::release() {
assert(identity() != NULL, "invariant");
Atomic::release_store(&_identity, (const void*)NULL);
@ -260,13 +265,11 @@ bool JfrBuffer::retired() const {
}
void JfrBuffer::set_retired() {
assert(acquired_by_self(), "invariant");
set(&_flags, RETIRED);
}
void JfrBuffer::clear_retired() {
if (retired()) {
assert(identity() != NULL, "invariant");
clear(&_flags, RETIRED);
}
}

36
src/hotspot/share/jfr/recorder/storage/jfrBuffer.hpp
View File

@ -60,9 +60,9 @@
//
class JfrBuffer {
public:
JfrBuffer* _next; // list support
private:
JfrBuffer* _next;
JfrBuffer* _prev;
const void* _identity;
u1* _pos;
mutable const u1* _top;
@ -77,22 +77,6 @@ class JfrBuffer {
bool initialize(size_t header_size, size_t size);
void reinitialize(bool exclusion = false);
JfrBuffer* next() const {
return _next;
}
JfrBuffer* prev() const {
return _prev;
}
void set_next(JfrBuffer* next) {
_next = next;
}
void set_prev(JfrBuffer* prev) {
_prev = prev;
}
const u1* start() const {
return ((const u1*)this) + _header_size;
}
@ -157,6 +141,9 @@ class JfrBuffer {
return Atomic::load_acquire(&_identity);
}
// use only if implied owner already
void set_identity(const void* id);
void acquire(const void* id);
bool try_acquire(const void* id);
bool acquired_by(const void* id) const;
@ -183,17 +170,4 @@ class JfrBuffer {
void clear_excluded();
};
class JfrAgeNode : public JfrBuffer {
private:
JfrBuffer* _retired;
public:
JfrAgeNode() : _retired(NULL) {}
void set_retired_buffer(JfrBuffer* retired) {
_retired = retired;
}
JfrBuffer* retired_buffer() const {
return _retired;
}
};
#endif // SHARE_JFR_RECORDER_STORAGE_JFRBUFFER_HPP

68
src/hotspot/share/jfr/recorder/storage/jfrFullStorage.hpp Normal file
View File

@ -0,0 +1,68 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_JFR_RECORDER_STORAGE_JFRFULLSTORAGE_HPP
#define SHARE_JFR_RECORDER_STORAGE_JFRFULLSTORAGE_HPP
#include "jfr/utilities/jfrAllocation.hpp"
#include "jfr/utilities/jfrConcurrentQueue.hpp"
class JfrStorageControl;
/*
* For full storage management.
*
* In essence, full storage is added to a FIFO queue, where the insertion order
* is used to represent the "is older" relation. Removes oldest data first.
*
* FullType the type of the data value to be stored in the list.
*
* NodeType template class for the node to store a value of FullType.
*
* AllocPolicy memory alloction.
*/
template <typename FullType, template <typename> class NodeType, typename AllocPolicy = JfrCHeapObj>
class JfrFullStorage : public AllocPolicy {
public:
typedef FullType Value;
typedef NodeType<Value>* NodePtr;
typedef NodeType<Value> Node;
JfrFullStorage(JfrStorageControl& control);
~JfrFullStorage();
bool initialize(size_t free_list_prealloc_count);
bool is_empty() const;
bool is_nonempty() const;
bool add(Value value);
Value remove();
template <typename Callback>
void iterate(Callback& cb);
private:
JfrStorageControl& _control;
JfrConcurrentQueue<Node, AllocPolicy>* _free_node_list;
JfrConcurrentQueue<Node, AllocPolicy>* _queue;
NodePtr acquire();
void release(NodePtr node);
};
#endif // SHARE_JFR_RECORDER_STORAGE_JFRFULLSTORAGE_HPP

121
src/hotspot/share/jfr/recorder/storage/jfrFullStorage.inline.hpp Normal file
View File

@ -0,0 +1,121 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_JFR_RECORDER_STORAGE_JFRSTORAGEFULLLIST_INLINE_HPP
#define SHARE_JFR_RECORDER_STORAGE_JFRSTORAGEFULLLIST_INLINE_HPP
#include "jfr/recorder/storage/jfrStorageControl.hpp"
#include "jfr/recorder/storage/jfrFullStorage.hpp"
#include "jfr/utilities/jfrConcurrentQueue.inline.hpp"
template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
JfrFullStorage<ValueType, NodeType, AllocPolicy>
::JfrFullStorage(JfrStorageControl& control) : _control(control), _free_node_list(NULL), _queue(NULL) {}
template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
JfrFullStorage<ValueType, NodeType, AllocPolicy>::~JfrFullStorage() {
NodePtr node;
while (_free_node_list->is_nonempty()) {
node = _free_node_list->remove();
delete node;
}
while (_queue->is_nonempty()) {
node = _queue->remove();
delete node;
}
}
template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
bool JfrFullStorage<ValueType, NodeType, AllocPolicy>::initialize(size_t free_list_prealloc_count) {
assert(_free_node_list == NULL, "invariant");
_free_node_list = new JfrConcurrentQueue<Node>();
if (_free_node_list == NULL || !_free_node_list->initialize()) {
return false;
}
for (size_t i = 0; i < free_list_prealloc_count; ++i) {
NodePtr node = new (ResourceObj::C_HEAP, mtTracing) Node();
if (node == NULL) {
return false;
}
_free_node_list->add(node);
}
assert(_queue == NULL, "invariant");
_queue = new JfrConcurrentQueue<Node>();
return _queue != NULL && _queue->initialize();
}
template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
inline bool JfrFullStorage<ValueType, NodeType, AllocPolicy>::is_empty() const {
return _queue->is_empty();
}
template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
inline bool JfrFullStorage<ValueType, NodeType, AllocPolicy>::is_nonempty() const {
return !is_empty();
}
template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
inline typename JfrFullStorage<ValueType, NodeType, AllocPolicy>::NodePtr
JfrFullStorage<ValueType, NodeType, AllocPolicy>::acquire() {
NodePtr node = _free_node_list->remove();
return node != NULL ? node : new (ResourceObj::C_HEAP, mtTracing) Node();
}
template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
inline void JfrFullStorage<ValueType, NodeType, AllocPolicy>
::release(typename JfrFullStorage<ValueType, NodeType, AllocPolicy>::NodePtr node) {
assert(node != NULL, "invariant");
_free_node_list->add(node);
}
template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
inline bool JfrFullStorage<ValueType, NodeType, AllocPolicy>::add(ValueType value) {
assert(value != NULL, "invariant");
NodePtr node = acquire();
assert(node != NULL, "invariant");
node->set_value(value);
const bool notify = _control.increment_full();
_queue->add(node);
return notify;
}
template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
inline ValueType JfrFullStorage<ValueType, NodeType, AllocPolicy>::remove() {
Value value = NULL;
NodePtr node = _queue->remove();
if (node != NULL) {
_control.decrement_full();
value = node->value();
release(node);
}
return value;
}
template <typename ValueType, template <typename> class NodeType, typename AllocPolicy>
template <typename Callback>
void JfrFullStorage<ValueType, NodeType, AllocPolicy>::iterate(Callback& cb) {
_queue->iterate(cb);
}
#endif // SHARE_JFR_RECORDER_STORAGE_JFRSTORAGEFULLLIST_INLINE_HPP

98
src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -25,82 +25,68 @@
#define SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACE_HPP
#include "jfr/utilities/jfrAllocation.hpp"
#include "jfr/utilities/jfrDoublyLinkedList.hpp"
#include "jfr/utilities/jfrIterator.hpp"
template <typename T, template <typename> class RetrievalType, typename Callback>
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType = FreeListType>
class JfrMemorySpace : public JfrCHeapObj {
public:
typedef T Type;
typedef RetrievalType<JfrMemorySpace<T, RetrievalType, Callback> > Retrieval;
typedef JfrDoublyLinkedList<Type> List;
typedef StopOnNullIterator<List> Iterator;
typedef FreeListType FreeList;
typedef FullListType FullList;
typedef typename FreeListType::Node Node;
typedef typename FreeListType::NodePtr NodePtr;
private:
List _free;
List _full;
size_t _min_elem_size;
size_t _limit_size;
size_t _cache_count;
FreeList _free_list;
FullList _full_list;
const size_t _min_elem_size;
const size_t _limit_size;
const size_t _free_list_cache_count;
size_t _free_list_count;
Callback* _callback;
bool should_populate_cache() const { return _free.count() < _cache_count; }
bool should_populate_free_list() const;
public:
JfrMemorySpace(size_t min_elem_size, size_t limit_size, size_t cache_count, Callback* callback);
JfrMemorySpace(size_t min_elem_size, size_t limit_size, size_t free_list_cache_count, Callback* callback);
~JfrMemorySpace();
bool initialize();
size_t min_elem_size() const { return _min_elem_size; }
size_t limit_size() const { return _limit_size; }
size_t min_elem_size() const;
size_t limit_size() const;
bool has_full() const { return _full.head() != NULL; }
bool has_free() const { return _free.head() != NULL; }
bool is_full_empty() const { return !has_full(); }
bool is_free_empty() const { return !has_free(); }
NodePtr allocate(size_t size);
void deallocate(NodePtr node);
size_t full_count() const { return _full.count(); }
size_t free_count() const { return _free.count(); }
NodePtr acquire(Thread* thread, size_t size = 0);
void release(NodePtr node);
List& full() { return _full; }
const List& full() const { return _full; }
List& free() { return _free; }
const List& free() const { return _free; }
FreeList& free_list();
const FreeList& free_list() const;
Type* full_head() { return _full.head(); }
Type* full_tail() { return _full.tail(); }
Type* free_head() { return _free.head(); }
Type* free_tail() { return _free.tail(); }
FullList& full_list();
const FullList& full_list() const;
void insert_free_head(Type* t) { _free.prepend(t); }
void insert_free_tail(Type* t) { _free.append(t); }
void insert_free_tail(Type* t, Type* tail, size_t count) { _free.append_list(t, tail, count); }
void insert_full_head(Type* t) { _full.prepend(t); }
void insert_full_tail(Type* t) { _full.append(t); }
void insert_full_tail(Type* t, Type* tail, size_t count) { _full.append_list(t, tail, count); }
bool free_list_is_empty() const;
bool full_list_is_empty() const;
bool free_list_is_nonempty() const;
bool full_list_is_nonempty() const;
bool in_free_list(const Node* node) const;
bool in_full_list(const Node* node) const;
bool in_mspace(const Node* node) const;
Type* remove_free(Type* t) { return _free.remove(t); }
Type* remove_full(Type* t) { return _full.remove(t); }
Type* remove_free_tail() { _free.remove(_free.tail()); }
Type* remove_full_tail() { return _full.remove(_full.tail()); }
Type* clear_full(bool return_tail = false) { return _full.clear(return_tail); }
Type* clear_free(bool return_tail = false) { return _free.clear(return_tail); }
void release_full(Type* t);
void release_free(Type* t);
void add_to_free_list(NodePtr node);
void add_to_full_list(NodePtr node);
void register_full(Type* t, Thread* thread) { _callback->register_full(t, thread); }
void lock() { _callback->lock(); }
void unlock() { _callback->unlock(); }
DEBUG_ONLY(bool is_locked() const { return _callback->is_locked(); })
NodePtr remove_from_free_list();
NodePtr remove_from_full_list();
Type* allocate(size_t size);
void deallocate(Type* t);
Type* get(size_t size, Thread* thread) { return Retrieval::get(size, this, thread); }
NodePtr clear_free_list();
NodePtr clear_full_list();
template <typename IteratorCallback, typename IteratorType>
void iterate(IteratorCallback& callback, bool full = true, jfr_iter_direction direction = forward);
template <typename Processor>
void iterate(Processor& processor, bool full_list = true);
bool in_full_list(const Type* t) const { return _full.in_list(t); }
bool in_free_list(const Type* t) const { return _free.in_list(t); }
void decrement_free_list_count();
void register_full(NodePtr node, Thread* thread);
};
#endif // SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACE_HPP

606
src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -28,48 +28,117 @@
#include "jfr/recorder/storage/jfrMemorySpace.hpp"
#include "runtime/os.hpp"
template <typename T, template <typename> class RetrievalType, typename Callback>
JfrMemorySpace<T, RetrievalType, Callback>::
JfrMemorySpace(size_t min_elem_size, size_t limit_size, size_t cache_count, Callback* callback) :
_free(),
_full(),
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::
JfrMemorySpace(size_t min_elem_size, size_t limit_size, size_t free_list_cache_count, Callback* callback) :
_free_list(),
_full_list(),
_min_elem_size(min_elem_size),
_limit_size(limit_size),
_cache_count(cache_count),
_free_list_cache_count(free_list_cache_count),
_free_list_count(0),
_callback(callback) {}
template <typename T, template <typename> class RetrievalType, typename Callback>
JfrMemorySpace<T, RetrievalType, Callback>::~JfrMemorySpace() {
Iterator full_iter(_full);
while (full_iter.has_next()) {
Type* t = full_iter.next();
_full.remove(t);
deallocate(t);
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::~JfrMemorySpace() {
while (full_list_is_nonempty()) {
NodePtr node = remove_from_full_list();
deallocate(node);
}
Iterator free_iter(_free);
while (free_iter.has_next()) {
Type* t = free_iter.next();
_free.remove(t);
deallocate(t);
while (free_list_is_nonempty()) {
NodePtr node = remove_from_free_list();
deallocate(node);
}
}
template <typename T, template <typename> class RetrievalType, typename Callback>
bool JfrMemorySpace<T, RetrievalType, Callback>::initialize() {
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
bool JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::initialize() {
if (!(_free_list.initialize() && _full_list.initialize())) {
return false;
}
assert(_min_elem_size % os::vm_page_size() == 0, "invariant");
assert(_limit_size % os::vm_page_size() == 0, "invariant");
// pre-allocate cache elements
for (size_t i = 0; i < _cache_count; ++i) {
Type* const t = allocate(_min_elem_size);
if (t == NULL) {
// pre-allocate free list cache elements
for (size_t i = 0; i < _free_list_cache_count; ++i) {
NodePtr const node = allocate(_min_elem_size);
if (node == NULL) {
return false;
}
insert_free_head(t);
add_to_free_list(node);
}
assert(_free.count() == _cache_count, "invariant");
return true;
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline bool JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::should_populate_free_list() const {
return _free_list_count < _free_list_cache_count;
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline size_t JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::min_elem_size() const {
return _min_elem_size;
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline size_t JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::limit_size() const {
return _limit_size;
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline FreeListType& JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::free_list() {
return _free_list;
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline const FreeListType& JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::free_list() const {
return _free_list;
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline FullListType& JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::full_list() {
return _full_list;
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline const FullListType& JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::full_list() const {
return _full_list;
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline bool JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::free_list_is_empty() const {
return _free_list.is_empty();
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline bool JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::free_list_is_nonempty() const {
return !free_list_is_empty();
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline bool JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::full_list_is_empty() const {
return _full_list.is_empty();
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline bool JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::full_list_is_nonempty() const {
return !full_list_is_empty();
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
bool JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::in_free_list(const typename FreeListType::Node* node) const {
return _free_list.in_list(node);
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
bool JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::in_full_list(const typename FreeListType::Node* node) const {
return _full_list.in_list(node);
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
bool JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::in_mspace(const typename FreeListType::Node* node) const {
return in_full_list(node) || in_free_list(node);
}
// allocations are even multiples of the mspace min size
static inline size_t align_allocation_size(size_t requested_size, size_t min_elem_size) {
assert((int)min_elem_size % os::vm_page_size() == 0, "invariant");
@ -81,88 +150,120 @@ static inline size_t align_allocation_size(size_t requested_size, size_t min_ele
return (size_t)alloc_size_bytes;
}
template <typename T, template <typename> class RetrievalType, typename Callback>
inline T* JfrMemorySpace<T, RetrievalType, Callback>::allocate(size_t size) {
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline typename FreeListType::NodePtr JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::allocate(size_t size) {
const size_t aligned_size_bytes = align_allocation_size(size, _min_elem_size);
void* const allocation = JfrCHeapObj::new_array<u1>(aligned_size_bytes + sizeof(T));
void* const allocation = JfrCHeapObj::new_array<u1>(aligned_size_bytes + sizeof(Node));
if (allocation == NULL) {
return NULL;
}
T* const t = new (allocation) T;
assert(t != NULL, "invariant");
if (!t->initialize(sizeof(T), aligned_size_bytes)) {
JfrCHeapObj::free(t, aligned_size_bytes + sizeof(T));
NodePtr node = new (allocation) Node();
assert(node != NULL, "invariant");
if (!node->initialize(sizeof(Node), aligned_size_bytes)) {
JfrCHeapObj::free(node, aligned_size_bytes + sizeof(Node));
return NULL;
}
return t;
return node;
}
template <typename T, template <typename> class RetrievalType, typename Callback>
inline void JfrMemorySpace<T, RetrievalType, Callback>::deallocate(T* t) {
assert(t != NULL, "invariant");
assert(!_free.in_list(t), "invariant");
assert(!_full.in_list(t), "invariant");
assert(t != NULL, "invariant");
JfrCHeapObj::free(t, t->total_size());
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline void JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::deallocate(typename FreeListType::NodePtr node) {
assert(node != NULL, "invariant");
assert(!in_free_list(node), "invariant");
assert(!in_full_list(node), "invariant");
assert(node != NULL, "invariant");
JfrCHeapObj::free(node, node->total_size());
}
template <typename T, template <typename> class RetrievalType, typename Callback>
inline void JfrMemorySpace<T, RetrievalType, Callback>::release_full(T* t) {
assert(is_locked(), "invariant");
assert(t != NULL, "invariant");
assert(_full.in_list(t), "invariant");
remove_full(t);
assert(!_full.in_list(t), "invariant");
if (t->transient()) {
deallocate(t);
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline typename FreeListType::NodePtr JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::acquire(Thread* thread, size_t size /* 0 */) {
return RetrievalPolicy<JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType> >::acquire(this, thread, size);
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline void JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::release(typename FreeListType::NodePtr node) {
assert(node != NULL, "invariant");
if (node->transient()) {
deallocate(node);
return;
}
assert(t->empty(), "invariant");
assert(!t->retired(), "invariant");
assert(t->identity() == NULL, "invariant");
if (should_populate_cache()) {
assert(!_free.in_list(t), "invariant");
insert_free_head(t);
assert(node->empty(), "invariant");
assert(!node->retired(), "invariant");
assert(node->identity() == NULL, "invariant");
if (should_populate_free_list()) {
add_to_free_list(node);
} else {
deallocate(t);
deallocate(node);
}
}
template <typename T, template <typename> class RetrievalType, typename Callback>
inline void JfrMemorySpace<T, RetrievalType, Callback>::release_free(T* t) {
assert(is_locked(), "invariant");
assert(t != NULL, "invariant");
assert(_free.in_list(t), "invariant");
if (t->transient()) {
remove_free(t);
assert(!_free.in_list(t), "invariant");
deallocate(t);
return;
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline void JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::add_to_free_list(typename FreeListType::NodePtr node) {
assert(node != NULL, "invariant");
assert(!in_free_list(node), "invariant");
_free_list.add(node);
Atomic::inc(&_free_list_count);
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline void JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::add_to_full_list(typename FreeListType::NodePtr node) {
assert(node != NULL, "invariant");
_full_list.add(node);
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline typename FreeListType::NodePtr JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::remove_from_free_list() {
NodePtr node = _free_list.remove();
if (node != NULL) {
decrement_free_list_count();
}
assert(t->empty(), "invariant");
assert(!t->retired(), "invariant");
assert(!t->excluded(), "invariant");
assert(t->identity() == NULL, "invariant");
if (!should_populate_cache()) {
remove_free(t);
assert(!_free.in_list(t), "invariant");
deallocate(t);
return node;
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline typename FreeListType::NodePtr JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::remove_from_full_list() {
return _full_list.remove();
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline void JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::decrement_free_list_count() {
Atomic::dec(&_free_list_count);
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline typename FreeListType::NodePtr JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::clear_free_list() {
NodePtr node = _free_list.clear();
NodePtr temp = node;
while (temp != NULL) {
decrement_free_list_count();
temp = temp->next();
}
return node;
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline typename FreeListType::NodePtr JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::clear_full_list() {
return _full_list.clear();
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
template <typename Processor>
inline void JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::iterate(Processor& processor, bool full_list /* true */) {
if (full_list) {
_full_list.iterate(processor);
} else {
_free_list.iterate(processor);
}
}
template <typename T, template <typename> class RetrievalType, typename Callback>
template <typename IteratorCallback, typename IteratorType>
inline void JfrMemorySpace<T, RetrievalType, Callback>
::iterate(IteratorCallback& callback, bool full, jfr_iter_direction direction) {
IteratorType iterator(full ? _full : _free, direction);
while (iterator.has_next()) {
callback.process(iterator.next());
}
template <typename Callback, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType>
inline void JfrMemorySpace<Callback, RetrievalPolicy, FreeListType, FullListType>::register_full(typename FreeListType::NodePtr node, Thread* thread) {
_callback->register_full(node, thread);
}
template <typename Mspace, typename Callback>
static inline Mspace* create_mspace(size_t buffer_size, size_t limit, size_t cache_count, Callback* cb) {
Mspace* const mspace = new Mspace(buffer_size, limit, cache_count, cb);
static inline Mspace* create_mspace(size_t min_elem_size, size_t limit, size_t free_list_cache_count, Callback* cb) {
Mspace* const mspace = new Mspace(min_elem_size, limit, free_list_cache_count, cb);
if (mspace != NULL) {
mspace->initialize();
}
@ -170,286 +271,231 @@ static inline Mspace* create_mspace(size_t buffer_size, size_t limit, size_t cac
}
template <typename Mspace>
inline size_t size_adjustment(size_t size, Mspace* mspace) {
assert(mspace != NULL, "invariant");
static const size_t min_elem_size = mspace->min_elem_size();
if (size < min_elem_size) {
size = min_elem_size;
}
return size;
inline typename Mspace::NodePtr mspace_allocate(size_t size, Mspace* mspace) {
return mspace->allocate(size);
}
template <typename Mspace>
inline typename Mspace::Type* mspace_allocate(size_t size, Mspace* mspace) {
return mspace->allocate(size_adjustment(size, mspace));
inline typename Mspace::NodePtr mspace_allocate_acquired(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::NodePtr node = mspace_allocate(size, mspace);
if (node == NULL) return NULL;
node->set_identity(thread);
return node;
}
template <typename Mspace>
inline typename Mspace::Type* mspace_allocate_acquired(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::Type* const t = mspace_allocate(size, mspace);
if (t == NULL) return NULL;
t->acquire(thread);
return t;
inline typename Mspace::NodePtr mspace_allocate_transient(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::NodePtr node = mspace_allocate_acquired(size, mspace, thread);
if (node == NULL) return NULL;
assert(node->acquired_by_self(), "invariant");
node->set_transient();
return node;
}
template <typename Mspace>
inline typename Mspace::Type* mspace_allocate_transient(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::Type* const t = mspace_allocate_acquired(size, mspace, thread);
if (t == NULL) return NULL;
assert(t->acquired_by_self(), "invariant");
t->set_transient();
return t;
inline typename Mspace::NodePtr mspace_allocate_transient_lease(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::NodePtr node = mspace_allocate_transient(size, mspace, thread);
if (node == NULL) return NULL;
assert(node->transient(), "invariant");
node->set_lease();
return node;
}
template <typename Mspace>
inline typename Mspace::Type* mspace_allocate_transient_lease(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::Type* const t = mspace_allocate_transient(size, mspace, thread);
if (t == NULL) return NULL;
assert(t->acquired_by_self(), "invariant");
assert(t->transient(), "invaiant");
t->set_lease();
return t;
inline typename Mspace::NodePtr mspace_allocate_to_full(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::NodePtr node = mspace_allocate_acquired(size, mspace, thread);
if (node == NULL) return NULL;
assert(node->acquired_by_self(), "invariant");
mspace->add_to_full_list(node);
return node;
}
template <typename Mspace>
inline typename Mspace::Type* mspace_allocate_to_full(size_t size, Mspace* mspace, Thread* thread) {
assert(mspace->is_locked(), "invariant");
typename Mspace::Type* const t = mspace_allocate_acquired(size, mspace, thread);
if (t == NULL) return NULL;
mspace->insert_full_head(t);
return t;
inline typename Mspace::NodePtr mspace_allocate_transient_to_full(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::NodePtr node = mspace_allocate_transient(size, mspace, thread);
if (node == NULL) return NULL;
assert(node->transient(), "invariant");
mspace->add_to_full_list(node);
return node;
}
template <typename Mspace>
class MspaceLock {
private:
Mspace* _mspace;
public:
MspaceLock(Mspace* mspace) : _mspace(mspace) { _mspace->lock(); }
~MspaceLock() { _mspace->unlock(); }
};
template <typename Mspace>
inline typename Mspace::Type* mspace_allocate_transient_to_full(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::Type* const t = mspace_allocate_transient(size, mspace, thread);
if (t == NULL) return NULL;
MspaceLock<Mspace> lock(mspace);
mspace->insert_full_head(t);
return t;
inline typename Mspace::NodePtr mspace_allocate_transient_lease_to_full(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::NodePtr node = mspace_allocate_transient_lease(size, mspace, thread);
if (node == NULL) return NULL;
assert(node->lease(), "invariant");
mspace->add_to_full_list(node);
return node;
}
template <typename Mspace>
inline typename Mspace::Type* mspace_allocate_transient_lease_to_full(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::Type* const t = mspace_allocate_transient_lease(size, mspace, thread);
if (t == NULL) return NULL;
assert(t->acquired_by_self(), "invariant");
assert(t->transient(), "invaiant");
assert(t->lease(), "invariant");
MspaceLock<Mspace> lock(mspace);
mspace->insert_full_head(t);
return t;
inline typename Mspace::NodePtr mspace_allocate_transient_lease_to_free(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::NodePtr node = mspace_allocate_transient_lease(size, mspace, thread);
if (node == NULL) return NULL;
assert(node->lease(), "invariant");
mspace->add_to_free_list(node);
return node;
}
template <typename Mspace>
inline typename Mspace::Type* mspace_allocate_transient_lease_to_free(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::Type* const t = mspace_allocate_transient_lease(size, mspace, thread);
if (t == NULL) return NULL;
assert(t->acquired_by_self(), "invariant");
assert(t->transient(), "invaiant");
assert(t->lease(), "invariant");
MspaceLock<Mspace> lock(mspace);
mspace->insert_free_head(t);
return t;
inline typename Mspace::NodePtr mspace_get_free(size_t size, Mspace* mspace, Thread* thread) {
return mspace->acquire(thread, size);
}
template <typename Mspace>
inline typename Mspace::Type* mspace_get_free(size_t size, Mspace* mspace, Thread* thread) {
return mspace->get(size, thread);
}
template <typename Mspace>
inline typename Mspace::Type* mspace_get_free_with_retry(size_t size, Mspace* mspace, size_t retry_count, Thread* thread) {
inline typename Mspace::NodePtr mspace_get_free_with_retry(size_t size, Mspace* mspace, size_t retry_count, Thread* thread) {
assert(size <= mspace->min_elem_size(), "invariant");
for (size_t i = 0; i < retry_count; ++i) {
typename Mspace::Type* const t = mspace_get_free(size, mspace, thread);
if (t != NULL) {
return t;
typename Mspace::NodePtr node = mspace_get_free(size, mspace, thread);
if (node != NULL) {
return node;
}
}
return NULL;
}
template <typename Mspace>
inline typename Mspace::Type* mspace_get_free_with_detach(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::Type* t = mspace_get_free(size, mspace, thread);
if (t != NULL) {
mspace->remove_free(t);
inline typename Mspace::NodePtr mspace_get_free_lease_with_retry(size_t size, Mspace* mspace, size_t retry_count, Thread* thread) {
typename Mspace::NodePtr node = mspace_get_free_with_retry(size, mspace, retry_count, thread);
if (node != NULL) {
node->set_lease();
}
return t;
return node;
}
template <typename Mspace>
inline typename Mspace::Type* mspace_get_free_to_full(size_t size, Mspace* mspace, Thread* thread) {
inline typename Mspace::NodePtr mspace_get_free_to_full(size_t size, Mspace* mspace, Thread* thread) {
assert(size <= mspace->min_elem_size(), "invariant");
assert(mspace->is_locked(), "invariant");
typename Mspace::Type* t = mspace_get_free(size, mspace, thread);
if (t == NULL) {
typename Mspace::NodePtr node = mspace_get_free(size, mspace, thread);
if (node == NULL) {
return NULL;
}
assert(t->acquired_by_self(), "invariant");
move_to_head(t, mspace->free(), mspace->full());
return t;
assert(node->acquired_by_self(), "invariant");
assert(!mspace->in_free_list(node), "invariant");
mspace->add_to_full_list(node);
return node;
}
template <typename Mspace>
inline typename Mspace::Type* mspace_get_to_full(size_t size, Mspace* mspace, Thread* thread) {
size = size_adjustment(size, mspace);
MspaceLock<Mspace> lock(mspace);
inline typename Mspace::NodePtr mspace_get_to_full(size_t size, Mspace* mspace, Thread* thread) {
if (size <= mspace->min_elem_size()) {
typename Mspace::Type* const t = mspace_get_free_to_full(size, mspace, thread);
if (t != NULL) {
return t;
typename Mspace::NodePtr node = mspace_get_free_to_full(size, mspace, thread);
if (node != NULL) {
return node;
}
}
return mspace_allocate_to_full(size, mspace, thread);
}
template <typename Mspace>
inline typename Mspace::Type* mspace_get_free_lease_with_retry(size_t size, Mspace* mspace, size_t retry_count, Thread* thread) {
typename Mspace::Type* t = mspace_get_free_with_retry(size, mspace, retry_count, thread);
if (t != NULL) {
t->set_lease();
}
return t;
}
template <typename Mspace>
inline typename Mspace::Type* mspace_get_lease(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::Type* t;
t = mspace_get_free_lease(size, mspace, thread);
if (t != NULL) {
assert(t->acquired_by_self(), "invariant");
assert(t->lease(), "invariant");
return t;
}
t = mspace_allocate_transient_to_full(size, mspace, thread);
if (t != NULL) {
t->set_lease();
}
return t;
}
template <typename Mspace>
inline void mspace_release_full(typename Mspace::Type* t, Mspace* mspace) {
assert(t != NULL, "invariant");
assert(t->unflushed_size() == 0, "invariant");
inline void mspace_release(typename Mspace::NodePtr node, Mspace* mspace) {
assert(node != NULL, "invariant");
assert(node->unflushed_size() == 0, "invariant");
assert(mspace != NULL, "invariant");
assert(mspace->is_locked(), "invariant");
mspace->release_full(t);
}
template <typename Mspace>
inline void mspace_release_free(typename Mspace::Type* t, Mspace* mspace) {
assert(t != NULL, "invariant");
assert(t->unflushed_size() == 0, "invariant");
assert(mspace != NULL, "invariant");
assert(mspace->is_locked(), "invariant");
mspace->release_free(t);
}
template <typename Mspace>
inline void mspace_release_full_critical(typename Mspace::Type* t, Mspace* mspace) {
MspaceLock<Mspace> lock(mspace);
mspace_release_full(t, mspace);
}
template <typename Mspace>
inline void mspace_release_free_critical(typename Mspace::Type* t, Mspace* mspace) {
MspaceLock<Mspace> lock(mspace);
mspace_release_free(t, mspace);
}
template <typename List>
inline void move_to_head(typename List::Node* t, List& from, List& to) {
assert(from.in_list(t), "invariant");
to.prepend(from.remove(t));
}
template <typename Processor, typename Mspace, typename Iterator>
inline void process_free_list_iterator_control(Processor& processor, Mspace* mspace, jfr_iter_direction direction = forward) {
mspace->template iterate<Processor, Iterator>(processor, false, direction);
}
template <typename Processor, typename Mspace, typename Iterator>
inline void process_full_list_iterator_control(Processor& processor, Mspace* mspace, jfr_iter_direction direction = forward) {
mspace->template iterate<Processor, Iterator>(processor, true, direction);
mspace->release(node);
}
template <typename Processor, typename Mspace>
inline void process_full_list(Processor& processor, Mspace* mspace, jfr_iter_direction direction = forward) {
inline void process_full_list(Processor& processor, Mspace* mspace) {
assert(mspace != NULL, "invariant");
if (mspace->is_full_empty()) return;
process_full_list_iterator_control<Processor, Mspace, typename Mspace::Iterator>(processor, mspace, direction);
if (mspace->full_list_is_nonempty()) {
mspace->iterate(processor);
}
}
template <typename Processor, typename Mspace>
inline void process_free_list(Processor& processor, Mspace* mspace, jfr_iter_direction direction = forward) {
inline void process_free_list(Processor& processor, Mspace* mspace) {
assert(mspace != NULL, "invariant");
assert(mspace->has_free(), "invariant");
process_free_list_iterator_control<Processor, Mspace, typename Mspace::Iterator>(processor, mspace, direction);
assert(mspace->free_list_is_nonempty(), "invariant");
mspace->iterate(processor, false);
}
template <typename Mspace>
class ReleaseOp : public StackObj {
private:
Mspace* _mspace;
Thread* _thread;
bool _release_full;
public:
typedef typename Mspace::Type Type;
ReleaseOp(Mspace* mspace, Thread* thread, bool release_full = true) :
_mspace(mspace), _thread(thread), _release_full(release_full) {}
bool process(Type* t);
typedef typename Mspace::Node Node;
ReleaseOp(Mspace* mspace) : _mspace(mspace) {}
bool process(typename Mspace::NodePtr node);
size_t processed() const { return 0; }
};
template <typename Mspace>
inline bool ReleaseOp<Mspace>::process(typename Mspace::Type* t) {
assert(t != NULL, "invariant");
// assumes some means of exclusive access to t
if (t->transient()) {
if (_release_full) {
mspace_release_full_critical(t, _mspace);
} else {
mspace_release_free_critical(t, _mspace);
}
inline bool ReleaseOp<Mspace>::process(typename Mspace::NodePtr node) {
assert(node != NULL, "invariant");
// assumes some means of exclusive access to the node
if (node->transient()) {
// make sure the transient node is already detached
_mspace->release(node);
return true;
}
t->reinitialize();
if (t->identity() != NULL) {
assert(t->empty(), "invariant");
assert(!t->retired(), "invariant");
t->release(); // publish
node->reinitialize();
if (node->identity() != NULL) {
assert(node->empty(), "invariant");
assert(!node->retired(), "invariant");
node->release(); // publish
}
return true;
}
template <typename Mspace>
class ScavengingReleaseOp : public StackObj {
private:
Mspace* _mspace;
typename Mspace::FullList& _full_list;
typename Mspace::NodePtr _prev;
size_t _count;
size_t _amount;
public:
typedef typename Mspace::Node Node;
ScavengingReleaseOp(Mspace* mspace) :
_mspace(mspace), _full_list(mspace->full_list()), _prev(NULL), _count(0), _amount(0) {}
bool process(typename Mspace::NodePtr node);
size_t processed() const { return _count; }
size_t amount() const { return _amount; }
};
template <typename Mspace>
inline bool ScavengingReleaseOp<Mspace>::process(typename Mspace::NodePtr node) {
assert(node != NULL, "invariant");
if (node->retired()) {
_prev = _full_list.excise(_prev, node);
if (node->transient()) {
_mspace->deallocate(node);
return true;
}
assert(node->identity() != NULL, "invariant");
assert(node->empty(), "invariant");
assert(!node->lease(), "invariant");
assert(!node->excluded(), "invariant");
++_count;
_amount += node->total_size();
node->clear_retired();
node->release();
mspace_release(node, _mspace);
return true;
}
_prev = node;
return true;
}
#ifdef ASSERT
template <typename T>
inline void assert_migration_state(const T* old, const T* new_buffer, size_t used, size_t requested) {
template <typename Node>
inline void assert_migration_state(const Node* old, const Node* new_node, size_t used, size_t requested) {
assert(old != NULL, "invariant");
assert(new_buffer != NULL, "invariant");
assert(new_node != NULL, "invariant");
assert(old->pos() >= old->start(), "invariant");
assert(old->pos() + used <= old->end(), "invariant");
assert(new_buffer->free_size() >= (used + requested), "invariant");
assert(new_node->free_size() >= (used + requested), "invariant");
}
#endif // ASSERT
template <typename T>
inline void migrate_outstanding_writes(const T* old, T* new_buffer, size_t used, size_t requested) {
DEBUG_ONLY(assert_migration_state(old, new_buffer, used, requested);)
template <typename Node>
inline void migrate_outstanding_writes(const Node* old, Node* new_node, size_t used, size_t requested) {
DEBUG_ONLY(assert_migration_state(old, new_node, used, requested);)
if (used > 0) {
memcpy(new_buffer->pos(), old->pos(), used);
memcpy(new_node->pos(), old->pos(), used);
}
}

96
src/hotspot/share/jfr/recorder/storage/jfrMemorySpaceRetrieval.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -25,31 +25,26 @@
#ifndef SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACERETRIEVAL_HPP
#define SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACERETRIEVAL_HPP
#include "memory/allocation.hpp"
#include "jfr/recorder/repository/jfrChunkWriter.hpp"
#include "jfr/recorder/storage/jfrBuffer.hpp"
#include "jfr/utilities/jfrAllocation.hpp"
#include "jfr/utilities/jfrTypes.hpp"
#include "jfr/utilities/jfrIterator.hpp"
/*
* Some policy classes for getting mspace memory
*/
/* Some policy classes for getting mspace memory. */
template <typename Mspace>
class JfrMspaceRetrieval : AllStatic {
class JfrMspaceRetrieval {
public:
typedef typename Mspace::Type Type;
static Type* get(size_t size, Mspace* mspace, typename Mspace::Iterator& iterator, Thread* thread) {
typedef typename Mspace::Node Node;
static Node* acquire(Mspace* mspace, Thread* thread, size_t size) {
StopOnNullCondition<typename Mspace::FreeList> iterator(mspace->free_list());
while (iterator.has_next()) {
Type* const t = iterator.next();
if (t->retired()) continue;
if (t->try_acquire(thread)) {
assert(!t->retired(), "invariant");
if (t->free_size() >= size) {
return t;
Node* const node = iterator.next();
if (node->retired()) continue;
if (node->try_acquire(thread)) {
assert(!node->retired(), "invariant");
if (node->free_size() >= size) {
return node;
}
t->set_retired();
mspace->register_full(t, thread);
node->set_retired();
mspace->register_full(node, thread);
}
}
return NULL;
@ -57,57 +52,24 @@ class JfrMspaceRetrieval : AllStatic {
};
template <typename Mspace>
class JfrMspaceAlternatingRetrieval {
private:
// provides stochastic distribution over "deque" endpoints; racy is ok here
static bool _last_access;
class JfrMspaceRemoveRetrieval : AllStatic {
public:
typedef typename Mspace::Type Type;
static Type* get(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::Iterator iterator(mspace->free(), (_last_access = !_last_access) ? forward : backward);
return JfrMspaceRetrieval<Mspace>::get(size, mspace, iterator, thread);
}
};
template <typename Mspace>
bool JfrMspaceAlternatingRetrieval<Mspace>::_last_access = false;
template <typename Mspace>
class JfrMspaceSequentialRetrieval {
public:
typedef typename Mspace::Type Type;
static Type* get(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::Iterator iterator(mspace->free());
return JfrMspaceRetrieval<Mspace>::get(size, mspace, iterator, thread);
}
};
template <typename Mspace>
class JfrExclusiveRetrieval : AllStatic {
public:
typedef typename Mspace::Type Type;
static Type* get(size_t size, Mspace* mspace, typename Mspace::Iterator& iterator, Thread* thread) {
assert(mspace->is_locked(), "invariant");
if (iterator.has_next()) {
Type* const t = iterator.next();
assert(!t->retired(), "invariant");
assert(t->identity() == NULL, "invariant");
assert(t->free_size() >= size, "invariant");
t->acquire(thread);
return t;
typedef typename Mspace::Node Node;
static Node* acquire(Mspace* mspace, Thread* thread, size_t size) {
StopOnNullConditionRemoval<typename Mspace::FreeList> iterator(mspace->free_list());
// it is the iterator that removes the nodes
while (iterator.has_next()) {
Node* const node = iterator.next();
if (node == NULL) return NULL;
mspace->decrement_free_list_count();
assert(node->free_size() >= size, "invariant");
assert(!node->retired(), "invariant");
assert(node->identity() == NULL, "invariant");
node->set_identity(thread);
return node;
}
return NULL;
}
};
template <typename Mspace>
class JfrThreadLocalRetrieval {
public:
typedef typename Mspace::Type Type;
static Type* get(size_t size, Mspace* mspace, Thread* thread) {
typename Mspace::Iterator iterator(mspace->free(), forward);
return JfrExclusiveRetrieval<Mspace>::get(size, mspace, iterator, thread);
}
};
#endif // SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACERETRIEVAL_HPP

397
src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp
View File

@ -30,20 +30,23 @@
#include "jfr/recorder/repository/jfrChunkWriter.hpp"
#include "jfr/recorder/service/jfrOptionSet.hpp"
#include "jfr/recorder/service/jfrPostBox.hpp"
#include "jfr/recorder/storage/jfrFullStorage.inline.hpp"
#include "jfr/recorder/storage/jfrMemorySpace.inline.hpp"
#include "jfr/recorder/storage/jfrStorage.hpp"
#include "jfr/recorder/storage/jfrStorageControl.hpp"
#include "jfr/recorder/storage/jfrStorageUtils.inline.hpp"
#include "jfr/utilities/jfrIterator.hpp"
#include "jfr/utilities/jfrLinkedList.inline.hpp"
#include "jfr/utilities/jfrTime.hpp"
#include "jfr/writers/jfrNativeEventWriter.hpp"
#include "logging/log.hpp"
#include "runtime/atomic.hpp"
#include "runtime/mutexLocker.hpp"
#include "runtime/os.inline.hpp"
#include "runtime/safepoint.hpp"
#include "runtime/thread.hpp"
typedef JfrStorage::Buffer* BufferPtr;
typedef JfrStorage::BufferPtr BufferPtr;
static JfrStorage* _instance = NULL;
static JfrStorageControl* _control;
@ -69,8 +72,6 @@ JfrStorage::JfrStorage(JfrChunkWriter& chunkwriter, JfrPostBox& post_box) :
_control(NULL),
_global_mspace(NULL),
_thread_local_mspace(NULL),
_transient_mspace(NULL),
_age_mspace(NULL),
_chunkwriter(chunkwriter),
_post_box(post_box) {}
@ -84,11 +85,8 @@ JfrStorage::~JfrStorage() {
if (_thread_local_mspace != NULL) {
delete _thread_local_mspace;
}
if (_transient_mspace != NULL) {
delete _transient_mspace;
}
if (_age_mspace != NULL) {
delete _age_mspace;
if (_full_list != NULL) {
delete _full_list;
}
_instance = NULL;
}
@ -99,21 +97,10 @@ static const size_t thread_local_cache_count = 8;
static const size_t thread_local_scavenge_threshold = thread_local_cache_count / 2;
static const size_t transient_buffer_size_multiplier = 8; // against thread local buffer size
template <typename Mspace>
static Mspace* create_mspace(size_t buffer_size, size_t limit, size_t cache_count, JfrStorage* storage_instance) {
Mspace* mspace = new Mspace(buffer_size, limit, cache_count, storage_instance);
if (mspace != NULL) {
mspace->initialize();
}
return mspace;
}
bool JfrStorage::initialize() {
assert(_control == NULL, "invariant");
assert(_global_mspace == NULL, "invariant");
assert(_thread_local_mspace == NULL, "invariant");
assert(_transient_mspace == NULL, "invariant");
assert(_age_mspace == NULL, "invariant");
const size_t num_global_buffers = (size_t)JfrOptionSet::num_global_buffers();
assert(num_global_buffers >= in_memory_discard_threshold_delta, "invariant");
@ -133,16 +120,9 @@ bool JfrStorage::initialize() {
if (_thread_local_mspace == NULL) {
return false;
}
_transient_mspace = create_mspace<JfrStorageMspace>(thread_buffer_size * transient_buffer_size_multiplier, unlimited_mspace_size, 0, this);
if (_transient_mspace == NULL) {
return false;
}
_age_mspace = create_mspace<JfrStorageAgeMspace>(0 /* no extra size except header */, unlimited_mspace_size, num_global_buffers, this);
if (_age_mspace == NULL) {
return false;
}
control().set_scavenge_threshold(thread_local_scavenge_threshold);
return true;
// The full list will contain nodes pointing to retired global and transient buffers.
_full_list = new JfrFullList(*_control);
return _full_list != NULL && _full_list->initialize(num_global_buffers * 2);
}
JfrStorageControl& JfrStorage::control() {
@ -164,7 +144,7 @@ BufferPtr JfrStorage::acquire_thread_local(Thread* thread, size_t size /* 0 */)
}
BufferPtr JfrStorage::acquire_transient(size_t size, Thread* thread) {
BufferPtr buffer = mspace_allocate_transient_lease_to_full(size, instance()._transient_mspace, thread);
BufferPtr buffer = mspace_allocate_transient_lease(size, instance()._thread_local_mspace, thread);
if (buffer == NULL) {
log_allocation_failure("transient memory", size);
return NULL;
@ -178,24 +158,24 @@ BufferPtr JfrStorage::acquire_transient(size_t size, Thread* thread) {
static BufferPtr get_lease(size_t size, JfrStorageMspace* mspace, JfrStorage& storage_instance, size_t retry_count, Thread* thread) {
assert(size <= mspace->min_elem_size(), "invariant");
while (true) {
BufferPtr t = mspace_get_free_lease_with_retry(size, mspace, retry_count, thread);
if (t == NULL && storage_instance.control().should_discard()) {
BufferPtr buffer = mspace_get_free_lease_with_retry(size, mspace, retry_count, thread);
if (buffer == NULL && storage_instance.control().should_discard()) {
storage_instance.discard_oldest(thread);
continue;
}
return t;
return buffer;
}
}
static BufferPtr get_promotion_buffer(size_t size, JfrStorageMspace* mspace, JfrStorage& storage_instance, size_t retry_count, Thread* thread) {
assert(size <= mspace->min_elem_size(), "invariant");
while (true) {
BufferPtr t = mspace_get_free_with_retry(size, mspace, retry_count, thread);
if (t == NULL && storage_instance.control().should_discard()) {
BufferPtr buffer= mspace_get_free_with_retry(size, mspace, retry_count, thread);
if (buffer == NULL && storage_instance.control().should_discard()) {
storage_instance.discard_oldest(thread);
continue;
}
return t;
return buffer;
}
}
@ -300,88 +280,15 @@ void JfrStorage::release_large(BufferPtr buffer, Thread* thread) {
}
}
static JfrAgeNode* new_age_node(BufferPtr buffer, JfrStorageAgeMspace* age_mspace, Thread* thread) {
assert(buffer != NULL, "invariant");
assert(age_mspace != NULL, "invariant");
return mspace_allocate_transient(0, age_mspace, thread);
}
static void log_registration_failure(size_t unflushed_size) {
log_warning(jfr)("Unable to register a full buffer of " SIZE_FORMAT " bytes.", unflushed_size);
log_debug(jfr, system)("Cleared 1 full buffer of " SIZE_FORMAT " bytes.", unflushed_size);
}
static void handle_registration_failure(BufferPtr buffer) {
assert(buffer != NULL, "invariant");
assert(buffer->retired(), "invariant");
const size_t unflushed_size = buffer->unflushed_size();
buffer->reinitialize();
log_registration_failure(unflushed_size);
}
static JfrAgeNode* get_free_age_node(JfrStorageAgeMspace* age_mspace, Thread* thread) {
assert(JfrBuffer_lock->owned_by_self(), "invariant");
return mspace_get_free_with_detach(0, age_mspace, thread);
}
static bool insert_full_age_node(JfrAgeNode* age_node, JfrStorageAgeMspace* age_mspace, Thread* thread) {
assert(JfrBuffer_lock->owned_by_self(), "invariant");
assert(age_node != NULL, "invariant");
assert(age_node->acquired_by_self(), "invariant");
assert(age_node->retired_buffer()->retired(), "invariant");
age_node->release(); // drop identity claim on age node when inserting to full list
assert(age_node->identity() == NULL, "invariant");
age_mspace->insert_full_head(age_node);
return true;
}
static bool full_buffer_registration(BufferPtr buffer, JfrStorageAgeMspace* age_mspace, JfrStorageControl& control, Thread* thread) {
assert(buffer != NULL, "invariant");
assert(buffer->retired(), "invariant");
assert(age_mspace != NULL, "invariant");
MutexLocker lock(JfrBuffer_lock, Mutex::_no_safepoint_check_flag);
JfrAgeNode* age_node = get_free_age_node(age_mspace, thread);
if (age_node == NULL) {
age_node = new_age_node(buffer, age_mspace, thread);
if (age_node == NULL) {
return false;
}
}
assert(age_node != NULL, "invariant");
assert(age_node->acquired_by_self(), "invariant");
age_node->set_retired_buffer(buffer);
control.increment_full();
return insert_full_age_node(age_node, age_mspace, thread);
}
void JfrStorage::register_full(BufferPtr buffer, Thread* thread) {
assert(buffer != NULL, "invariant");
assert(buffer->retired(), "invariant");
assert(buffer->acquired_by(thread), "invariant");
if (!full_buffer_registration(buffer, _age_mspace, control(), thread)) {
handle_registration_failure(buffer);
}
if (control().should_post_buffer_full_message()) {
assert(buffer->retired(), "invariant");
if (_full_list->add(buffer)) {
_post_box.post(MSG_FULLBUFFER);
}
}
void JfrStorage::lock() {
assert(!JfrBuffer_lock->owned_by_self(), "invariant");
JfrBuffer_lock->lock_without_safepoint_check();
}
void JfrStorage::unlock() {
assert(JfrBuffer_lock->owned_by_self(), "invariant");
JfrBuffer_lock->unlock();
}
#ifdef ASSERT
bool JfrStorage::is_locked() const {
return JfrBuffer_lock->owned_by_self();
}
#endif
// don't use buffer on return, it is gone
void JfrStorage::release(BufferPtr buffer, Thread* thread) {
assert(buffer != NULL, "invariant");
@ -395,7 +302,7 @@ void JfrStorage::release(BufferPtr buffer, Thread* thread) {
}
assert(buffer->empty(), "invariant");
assert(buffer->identity() != NULL, "invariant");
control().increment_dead();
buffer->clear_excluded();
buffer->set_retired();
}
@ -403,16 +310,15 @@ void JfrStorage::release_thread_local(BufferPtr buffer, Thread* thread) {
assert(buffer != NULL, "invariant");
JfrStorage& storage_instance = instance();
storage_instance.release(buffer, thread);
if (storage_instance.control().should_scavenge()) {
storage_instance._post_box.post(MSG_DEADBUFFER);
}
}
static void log_discard(size_t count, size_t amount, size_t current) {
static void log_discard(size_t pre_full_count, size_t post_full_count, size_t amount) {
if (log_is_enabled(Debug, jfr, system)) {
assert(count > 0, "invariant");
log_debug(jfr, system)("Cleared " SIZE_FORMAT " full buffer(s) of " SIZE_FORMAT" bytes.", count, amount);
log_debug(jfr, system)("Current number of full buffers " SIZE_FORMAT "", current);
const size_t number_of_discards = pre_full_count - post_full_count;
if (number_of_discards > 0) {
log_debug(jfr, system)("Cleared " SIZE_FORMAT " full buffer(s) of " SIZE_FORMAT" bytes.", number_of_discards, amount);
log_debug(jfr, system)("Current number of full buffers " SIZE_FORMAT "", number_of_discards);
}
}
}
@ -423,34 +329,24 @@ void JfrStorage::discard_oldest(Thread* thread) {
return;
}
const size_t num_full_pre_discard = control().full_count();
size_t num_full_post_discard = 0;
size_t discarded_size = 0;
while (true) {
JfrAgeNode* const oldest_age_node = _age_mspace->full_tail();
if (oldest_age_node == NULL) {
break;
}
assert(oldest_age_node->identity() == NULL, "invariant");
BufferPtr const buffer = oldest_age_node->retired_buffer();
assert(buffer->retired(), "invariant");
assert(buffer->identity() != NULL, "invariant");
discarded_size += buffer->discard();
assert(buffer->unflushed_size() == 0, "invariant");
num_full_post_discard = control().decrement_full();
mspace_release_full(oldest_age_node, _age_mspace);
if (buffer->transient()) {
mspace_release_full(buffer, _transient_mspace);
while (_full_list->is_nonempty()) {
BufferPtr oldest = _full_list->remove();
assert(oldest != NULL, "invariant");
assert(oldest->identity() != NULL, "invariant");
discarded_size += oldest->discard();
assert(oldest->unflushed_size() == 0, "invariant");
if (oldest->transient()) {
mspace_release(oldest, _thread_local_mspace);
continue;
}
buffer->reinitialize();
buffer->release(); // publish
oldest->reinitialize();
assert(!oldest->retired(), "invariant");
oldest->release(); // publish
break;
}
JfrBuffer_lock->unlock();
const size_t number_of_discards = num_full_pre_discard - num_full_post_discard;
if (number_of_discards > 0) {
log_discard(number_of_discards, discarded_size, num_full_post_discard);
}
log_discard(num_full_pre_discard, control().full_count(), discarded_size);
}
}
@ -508,11 +404,9 @@ BufferPtr JfrStorage::flush_regular(BufferPtr cur, const u1* const cur_pos, size
// which is where the "used / uncommitted" data resides. It is therefore both
// possible and valid to migrate data after the flush. This is however only
// the case for stable thread local buffers; it is not the case for large buffers.
if (!cur->empty()) {
flush_regular_buffer(cur, t);
if (cur->excluded()) {
return cur;
}
flush_regular_buffer(cur, t);
if (cur->excluded()) {
return cur;
}
if (cur->free_size() >= req) {
// simplest case, no switching of buffers
@ -602,120 +496,64 @@ typedef UnBufferedWriteToChunk<JfrBuffer> WriteOperation;
typedef MutexedWriteOp<WriteOperation> MutexedWriteOperation;
typedef ConcurrentWriteOp<WriteOperation> ConcurrentWriteOperation;
typedef Retired<JfrBuffer, true> NonRetired;
typedef Excluded<JfrBuffer, true> NonExcluded;
typedef CompositeOperation<NonRetired, NonExcluded> BufferPredicate;
typedef PredicatedMutexedWriteOp<WriteOperation, BufferPredicate> ThreadLocalMutexedWriteOperation;
typedef PredicatedConcurrentWriteOp<WriteOperation, BufferPredicate> ThreadLocalConcurrentWriteOperation;
typedef PredicatedConcurrentWriteOp<WriteOperation, NonExcluded> ConcurrentWriteOperationNonExcluded;
typedef ReleaseOp<JfrThreadLocalMspace> ReleaseFullOperation;
typedef ScavengingReleaseOp<JfrThreadLocalMspace> ReleaseThreadLocalOperation;
typedef CompositeOperation<ConcurrentWriteOperationNonExcluded, ReleaseThreadLocalOperation> ConcurrentWriteThreadLocalOperationWithRelease;
size_t JfrStorage::write() {
const size_t full_elements = write_full();
WriteOperation wo(_chunkwriter);
NonRetired nr;
NonExcluded ne;
BufferPredicate bp(&nr, &ne);
ThreadLocalConcurrentWriteOperation tlwo(wo, bp);
process_full_list(tlwo, _thread_local_mspace);
ConcurrentWriteOperation cwo(wo);
process_free_list(cwo, _global_mspace);
ConcurrentWriteOperationNonExcluded cwone(wo, ne);
ReleaseThreadLocalOperation rtlo(_thread_local_mspace);
ConcurrentWriteThreadLocalOperationWithRelease tl_op(&cwone, &rtlo);
process_full_list(tl_op, _thread_local_mspace);
assert(_global_mspace->full_list_is_empty(), "invariant");
process_free_list(cwone, _global_mspace);
return full_elements + wo.elements();
}
size_t JfrStorage::write_at_safepoint() {
assert(SafepointSynchronize::is_at_safepoint(), "invariant");
WriteOperation wo(_chunkwriter);
MutexedWriteOperation writer(wo); // mutexed write mode
NonRetired nr;
NonExcluded ne;
BufferPredicate bp(&nr, &ne);
ThreadLocalMutexedWriteOperation tlmwo(wo, bp);
process_full_list(tlmwo, _thread_local_mspace);
assert(_transient_mspace->is_free_empty(), "invariant");
process_full_list(writer, _transient_mspace);
assert(_global_mspace->is_full_empty(), "invariant");
process_free_list(writer, _global_mspace);
ConcurrentWriteOperationNonExcluded cwone(wo, ne); // concurrent because of gc's
process_full_list(cwone, _thread_local_mspace);
assert(_global_mspace->full_list_is_empty(), "invariant");
process_free_list(cwone, _global_mspace);
return wo.elements();
}
typedef DiscardOp<DefaultDiscarder<JfrStorage::Buffer> > DiscardOperation;
typedef ReleaseOp<JfrStorageMspace> ReleaseOperation;
typedef CompositeOperation<MutexedWriteOperation, ReleaseOperation> FullOperation;
typedef CompositeOperation<DiscardOperation, ReleaseThreadLocalOperation> ThreadLocalDiscardOperation;
size_t JfrStorage::clear() {
const size_t full_elements = clear_full();
DiscardOperation discarder(concurrent); // concurrent discard mode
process_full_list(discarder, _thread_local_mspace);
assert(_transient_mspace->is_free_empty(), "invariant");
process_full_list(discarder, _transient_mspace);
assert(_global_mspace->is_full_empty(), "invariant");
ReleaseThreadLocalOperation rtlo(_thread_local_mspace);
ThreadLocalDiscardOperation tldo(&discarder, &rtlo);
process_full_list(tldo, _thread_local_mspace);
assert(_global_mspace->full_list_is_empty(), "invariant");
process_free_list(discarder, _global_mspace);
return full_elements + discarder.elements();
}
static void insert_free_age_nodes(JfrStorageAgeMspace* age_mspace, JfrAgeNode* head, JfrAgeNode* tail, size_t count) {
if (tail != NULL) {
assert(tail->next() == NULL, "invariant");
assert(head != NULL, "invariant");
assert(head->prev() == NULL, "invariant");
MutexLocker buffer_lock(JfrBuffer_lock, Mutex::_no_safepoint_check_flag);
age_mspace->insert_free_tail(head, tail, count);
}
}
template <typename Processor>
static void process_age_list(Processor& processor, JfrStorageAgeMspace* age_mspace, JfrAgeNode* head, size_t count) {
assert(age_mspace != NULL, "invariant");
assert(head != NULL, "invariant");
assert(count > 0, "invariant");
JfrAgeNode* node = head;
JfrAgeNode* last = NULL;
while (node != NULL) {
last = node;
assert(node->identity() == NULL, "invariant");
BufferPtr const buffer = node->retired_buffer();
assert(buffer != NULL, "invariant");
assert(buffer->retired(), "invariant");
processor.process(buffer);
// at this point, buffer is already live or destroyed
JfrAgeNode* const next = (JfrAgeNode*)node->next();
if (node->transient()) {
// detach
last = (JfrAgeNode*)last->prev();
if (last != NULL) {
last->set_next(next);
} else {
head = next;
}
if (next != NULL) {
next->set_prev(last);
}
--count;
age_mspace->deallocate(node);
}
node = next;
}
insert_free_age_nodes(age_mspace, head, last, count);
}
template <typename Processor>
static size_t process_full(Processor& processor, JfrStorageControl& control, JfrStorageAgeMspace* age_mspace) {
assert(age_mspace != NULL, "invariant");
if (age_mspace->is_full_empty()) {
// nothing to do
return 0;
}
size_t count;
JfrAgeNode* head;
{
// fetch age list
MutexLocker buffer_lock(JfrBuffer_lock, Mutex::_no_safepoint_check_flag);
count = age_mspace->full_count();
head = age_mspace->clear_full();
control.reset_full();
}
assert(head != NULL, "invariant");
assert(count > 0, "invariant");
process_age_list(processor, age_mspace, head, count);
static size_t process_full(Processor& processor, JfrFullList* list, JfrStorageControl& control) {
assert(list != NULL, "invariant");
assert(list->is_nonempty(), "invariant");
size_t count = 0;
do {
BufferPtr full = list->remove();
if (full == NULL) break;
assert(full->retired(), "invariant");
processor.process(full);
// at this point, the buffer is already live or destroyed
++count;
} while (list->is_nonempty());
return count;
}
@ -728,93 +566,36 @@ static void log(size_t count, size_t amount, bool clear = false) {
}
}
typedef CompositeOperation<MutexedWriteOperation, ReleaseFullOperation> FullOperation;
// full writer
// Assumption is retired only; exclusive access
// MutexedWriter -> ReleaseOp
//
size_t JfrStorage::write_full() {
assert(_chunkwriter.is_valid(), "invariant");
Thread* const thread = Thread::current();
WriteOperation wo(_chunkwriter);
MutexedWriteOperation writer(wo); // a retired buffer implies mutexed access
ReleaseOperation ro(_transient_mspace, thread);
FullOperation cmd(&writer, &ro);
const size_t count = process_full(cmd, control(), _age_mspace);
if (0 == count) {
assert(0 == writer.elements(), "invariant");
if (_full_list->is_empty()) {
return 0;
}
const size_t size = writer.size();
log(count, size);
WriteOperation wo(_chunkwriter);
MutexedWriteOperation writer(wo); // a retired buffer implies mutexed access
ReleaseFullOperation rfo(_thread_local_mspace);
FullOperation cmd(&writer, &rfo);
const size_t count = process_full(cmd, _full_list, control());
if (count != 0) {
log(count, writer.size());
}
return count;
}
size_t JfrStorage::clear_full() {
if (_full_list->is_empty()) {
return 0;
}
DiscardOperation discarder(mutexed); // a retired buffer implies mutexed access
const size_t count = process_full(discarder, control(), _age_mspace);
if (0 == count) {
assert(0 == discarder.elements(), "invariant");
return 0;
const size_t count = process_full(discarder, _full_list, control());
if (count != 0) {
log(count, discarder.size());
}
const size_t size = discarder.size();
log(count, size, true);
return count;
}
static void scavenge_log(size_t count, size_t amount, size_t current) {
if (count > 0) {
if (log_is_enabled(Debug, jfr, system)) {
log_debug(jfr, system)("Released " SIZE_FORMAT " dead buffer(s) of " SIZE_FORMAT" B of data.", count, amount);
log_debug(jfr, system)("Current number of dead buffers " SIZE_FORMAT "", current);
}
}
}
template <typename Mspace>
class Scavenger {
private:
JfrStorageControl& _control;
Mspace* _mspace;
size_t _count;
size_t _amount;
public:
typedef typename Mspace::Type Type;
Scavenger(JfrStorageControl& control, Mspace* mspace) : _control(control), _mspace(mspace), _count(0), _amount(0) {}
bool process(Type* t) {
if (t->retired()) {
assert(t->identity() != NULL, "invariant");
assert(t->empty(), "invariant");
assert(!t->transient(), "invariant");
assert(!t->lease(), "invariant");
++_count;
_amount += t->total_size();
if (t->excluded()) {
t->clear_excluded();
}
assert(!t->excluded(), "invariant");
t->clear_retired();
t->release();
_control.decrement_dead();
mspace_release_full_critical(t, _mspace);
}
return true;
}
size_t processed() const { return _count; }
size_t amount() const { return _amount; }
};
size_t JfrStorage::scavenge() {
JfrStorageControl& ctrl = control();
if (ctrl.dead_count() == 0) {
return 0;
}
Scavenger<JfrThreadLocalMspace> scavenger(ctrl, _thread_local_mspace);
process_full_list(scavenger, _thread_local_mspace);
const size_t count = scavenger.processed();
if (0 == count) {
assert(0 == scavenger.amount(), "invariant");
return 0;
}
scavenge_log(count, scavenger.amount(), ctrl.dead_count());
return count;
}

59
src/hotspot/share/jfr/recorder/storage/jfrStorage.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -25,52 +25,51 @@
#define SHARE_JFR_RECORDER_STORAGE_JFRSTORAGE_HPP
#include "jfr/recorder/storage/jfrBuffer.hpp"
#include "jfr/recorder/storage/jfrFullStorage.hpp"
#include "jfr/recorder/storage/jfrMemorySpace.hpp"
#include "jfr/recorder/storage/jfrMemorySpaceRetrieval.hpp"
#include "jfr/utilities/jfrConcurrentQueue.hpp"
#include "jfr/utilities/jfrLinkedList.hpp"
#include "jfr/utilities/jfrNode.hpp"
#include "jfr/utilities/jfrRelation.hpp"
class JfrChunkWriter;
class JfrPostBox;
class JfrStorage;
class JfrStorageControl;
typedef JfrMemorySpace<JfrBuffer, JfrMspaceAlternatingRetrieval, JfrStorage> JfrStorageMspace;
typedef JfrMemorySpace<JfrBuffer, JfrThreadLocalRetrieval, JfrStorage> JfrThreadLocalMspace;
typedef JfrMemorySpace<JfrAgeNode, JfrMspaceSequentialRetrieval, JfrStorage> JfrStorageAgeMspace;
typedef JfrMemorySpace<JfrStorage, JfrMspaceRetrieval, JfrLinkedList<JfrBuffer> > JfrStorageMspace;
typedef JfrMemorySpace<JfrStorage, JfrMspaceRemoveRetrieval, JfrConcurrentQueue<JfrBuffer>, JfrLinkedList<JfrBuffer> > JfrThreadLocalMspace;
typedef JfrFullStorage<JfrBuffer*, JfrValueNode> JfrFullList;
//
// Responsible for providing backing storage for writing events.
//
class JfrStorage : public JfrCHeapObj {
public:
typedef JfrStorageMspace::Type Buffer;
typedef JfrStorageMspace::Node Buffer;
typedef JfrStorageMspace::NodePtr BufferPtr;
private:
JfrStorageControl* _control;
JfrStorageMspace* _global_mspace;
JfrThreadLocalMspace* _thread_local_mspace;
JfrStorageMspace* _transient_mspace;
JfrStorageAgeMspace* _age_mspace;
JfrFullList* _full_list;
JfrChunkWriter& _chunkwriter;
JfrPostBox& _post_box;
// mspace callbacks
void register_full(Buffer* t, Thread* thread);
void lock();
void unlock();
DEBUG_ONLY(bool is_locked() const;)
Buffer* acquire_large(size_t size, Thread* t);
Buffer* acquire_transient(size_t size, Thread* thread);
bool flush_regular_buffer(Buffer* const buffer, Thread* t);
Buffer* flush_regular(Buffer* cur, const u1* cur_pos, size_t used, size_t req, bool native, Thread* t);
Buffer* flush_large(Buffer* cur, const u1* cur_pos, size_t used, size_t req, bool native, Thread* t);
Buffer* provision_large(Buffer* cur, const u1* cur_pos, size_t used, size_t req, bool native, Thread* t);
void release(Buffer* buffer, Thread* t);
BufferPtr acquire_large(size_t size, Thread* thread);
BufferPtr acquire_transient(size_t size, Thread* thread);
bool flush_regular_buffer(BufferPtr buffer, Thread* thread);
BufferPtr flush_regular(BufferPtr cur, const u1* cur_pos, size_t used, size_t req, bool native, Thread* thread);
BufferPtr flush_large(BufferPtr cur, const u1* cur_pos, size_t used, size_t req, bool native, Thread* thread);
BufferPtr provision_large(BufferPtr cur, const u1* cur_pos, size_t used, size_t req, bool native, Thread* thread);
void release(BufferPtr buffer, Thread* thread);
size_t clear();
size_t clear_full();
size_t write_full();
size_t write_at_safepoint();
size_t scavenge();
JfrStorage(JfrChunkWriter& cw, JfrPostBox& post_box);
~JfrStorage();
@ -80,19 +79,21 @@ class JfrStorage : public JfrCHeapObj {
bool initialize();
static void destroy();
public:
static Buffer* acquire_thread_local(Thread* t, size_t size = 0);
static void release_thread_local(Buffer* buffer, Thread* t);
void release_large(Buffer* const buffer, Thread* t);
static Buffer* flush(Buffer* cur, size_t used, size_t req, bool native, Thread* t);
void discard_oldest(Thread* t);
static JfrStorageControl& control();
// mspace callback
void register_full(BufferPtr buffer, Thread* thread);
public:
static BufferPtr acquire_thread_local(Thread* thread, size_t size = 0);
static void release_thread_local(BufferPtr buffer, Thread* thread);
void release_large(BufferPtr buffer, Thread* thread);
static BufferPtr flush(BufferPtr cur, size_t used, size_t req, bool native, Thread* thread);
void discard_oldest(Thread* thread);
static JfrStorageControl& control();
size_t write();
friend class JfrRecorder;
friend class JfrRecorderService;
template <typename, template <typename> class, typename>
template <typename, template <typename> class, typename, typename>
friend class JfrMemorySpace;
};

80
src/hotspot/share/jfr/recorder/storage/jfrStorageControl.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -25,40 +25,15 @@
#include "precompiled.hpp"
#include "jfr/recorder/storage/jfrStorageControl.hpp"
#include "runtime/atomic.hpp"
#include "runtime/mutexLocker.hpp"
// returns the updated value
static jlong atomic_add(size_t value, size_t volatile* const dest) {
size_t compare_value;
size_t exchange_value;
do {
compare_value = *dest;
exchange_value = compare_value + value;
} while (Atomic::cmpxchg(dest, compare_value, exchange_value) != compare_value);
return exchange_value;
}
static jlong atomic_dec(size_t volatile* const dest) {
size_t compare_value;
size_t exchange_value;
do {
compare_value = *dest;
assert(compare_value >= 1, "invariant");
exchange_value = compare_value - 1;
} while (Atomic::cmpxchg(dest, compare_value, exchange_value) != compare_value);
return exchange_value;
}
const size_t max_lease_factor = 2;
JfrStorageControl::JfrStorageControl(size_t global_count_total, size_t in_memory_discard_threshold) :
_global_count_total(global_count_total),
_full_count(0),
_global_lease_count(0),
_dead_count(0),
_to_disk_threshold(0),
_in_memory_discard_threshold(in_memory_discard_threshold),
_global_lease_threshold(global_count_total / max_lease_factor),
_scavenge_threshold(0),
_to_disk(false) {}
bool JfrStorageControl::to_disk() const {
@ -73,21 +48,24 @@ size_t JfrStorageControl::full_count() const {
return _full_count;
}
// mutexed access
size_t JfrStorageControl::increment_full() {
assert(JfrBuffer_lock->owned_by_self(), "invariant");
return ++_full_count;
bool JfrStorageControl::increment_full() {
const size_t result = Atomic::add(&_full_count, (size_t)1);
return to_disk() && result > _to_disk_threshold;
}
size_t JfrStorageControl::decrement_full() {
assert(JfrBuffer_lock->owned_by_self(), "invariant");
assert(_full_count > 0, "invariant");
return --_full_count;
size_t current;
size_t exchange;
do {
current = _full_count;
exchange = current - 1;
} while (Atomic::cmpxchg(&_full_count, current, exchange) != current);
return exchange;
}
void JfrStorageControl::reset_full() {
assert(JfrBuffer_lock->owned_by_self(), "invariant");
_full_count = 0;
Atomic::store(&_full_count, (size_t)0);
}
bool JfrStorageControl::should_post_buffer_full_message() const {
@ -98,42 +76,24 @@ bool JfrStorageControl::should_discard() const {
return !to_disk() && full_count() >= _in_memory_discard_threshold;
}
// concurrent with accuracy requirement
size_t JfrStorageControl::global_lease_count() const {
return Atomic::load(&_global_lease_count);
}
size_t JfrStorageControl::increment_leased() {
return atomic_add(1, &_global_lease_count);
return Atomic::add(&_global_lease_count, (size_t)1);
}
size_t JfrStorageControl::decrement_leased() {
return atomic_dec(&_global_lease_count);
size_t current;
size_t exchange;
do {
current = _global_lease_count;
exchange = current - 1;
} while (Atomic::cmpxchg(&_global_lease_count, current, exchange) != current);
return exchange;
}
bool JfrStorageControl::is_global_lease_allowed() const {
return global_lease_count() <= _global_lease_threshold;
}
// concurrent with lax requirement
size_t JfrStorageControl::dead_count() const {
return _dead_count;
}
size_t JfrStorageControl::increment_dead() {
return atomic_add(1, &_dead_count);
}
size_t JfrStorageControl::decrement_dead() {
return atomic_dec(&_dead_count);
}
bool JfrStorageControl::should_scavenge() const {
return dead_count() >= _scavenge_threshold;
}
void JfrStorageControl::set_scavenge_threshold(size_t number_of_dead_buffers) {
_scavenge_threshold = number_of_dead_buffers;
}

13
src/hotspot/share/jfr/recorder/storage/jfrStorageControl.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -32,11 +32,9 @@ class JfrStorageControl : public JfrCHeapObj {
size_t _global_count_total;
size_t _full_count;
volatile size_t _global_lease_count;
volatile size_t _dead_count;
size_t _to_disk_threshold;
size_t _in_memory_discard_threshold;
size_t _global_lease_threshold;
size_t _scavenge_threshold;
bool _to_disk;
public:
@ -46,7 +44,7 @@ class JfrStorageControl : public JfrCHeapObj {
bool to_disk() const;
size_t full_count() const;
size_t increment_full();
bool increment_full();
size_t decrement_full();
void reset_full();
bool should_post_buffer_full_message() const;
@ -56,13 +54,6 @@ class JfrStorageControl : public JfrCHeapObj {
size_t increment_leased();
size_t decrement_leased();
bool is_global_lease_allowed() const;
size_t dead_count() const;
size_t increment_dead();
size_t decrement_dead();
void set_scavenge_threshold(size_t number_of_dead_buffers);
bool should_scavenge() const;
};
#endif // SHARE_JFR_RECORDER_STORAGE_JFRSTORAGECONTROL_HPP

13
src/hotspot/share/jfr/recorder/storage/jfrStorageUtils.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -189,4 +189,15 @@ class DiscardOp {
size_t size() const { return _operation.size(); }
};
template <typename Operation>
class ExclusiveDiscardOp : private DiscardOp<Operation> {
public:
typedef typename Operation::Type Type;
ExclusiveDiscardOp(jfr_operation_mode mode = concurrent) : DiscardOp<Operation>(mode) {}
bool process(Type* t);
size_t processed() const { return DiscardOp<Operation>::processed(); }
size_t elements() const { return DiscardOp<Operation>::elements(); }
size_t size() const { return DiscardOp<Operation>::size(); }
};
#endif // SHARE_JFR_RECORDER_STORAGE_JFRSTORAGEUTILS_HPP

25
src/hotspot/share/jfr/recorder/storage/jfrStorageUtils.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -52,15 +52,24 @@ inline size_t get_unflushed_size(const u1* top, Type* t) {
template <typename Operation>
inline bool ConcurrentWriteOp<Operation>::process(typename Operation::Type* t) {
const bool is_retired = t->retired();
// acquire_critical_section_top() must be read before pos() for stable access
const u1* const top = t->acquire_critical_section_top();
const u1* const top = is_retired ? t->top() : t->acquire_critical_section_top();
const size_t unflushed_size = get_unflushed_size(top, t);
if (unflushed_size == 0) {
t->release_critical_section_top(top);
if (is_retired) {
t->set_top(top);
} else {
t->release_critical_section_top(top);
}
return true;
}
const bool result = _operation.write(t, top, unflushed_size);
t->release_critical_section_top(top + unflushed_size);
if (is_retired) {
t->set_top(top + unflushed_size);
} else {
t->release_critical_section_top(top + unflushed_size);
}
return result;
}
@ -119,4 +128,12 @@ inline bool DiscardOp<Operation>::process(typename Operation::Type* t) {
return result;
}
template <typename Operation>
inline bool ExclusiveDiscardOp<Operation>::process(typename Operation::Type* t) {
retired_sensitive_acquire(t);
assert(t->acquired_by_self() || t->retired(), "invariant");
// User is required to ensure proper release of the acquisition
return DiscardOp<Operation>::process(t);
}
#endif // SHARE_JFR_RECORDER_STORAGE_JFRSTORAGEUTILS_INLINE_HPP

93
src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -31,14 +31,14 @@
#include "jfr/recorder/storage/jfrStorageUtils.inline.hpp"
#include "jfr/recorder/stringpool/jfrStringPool.hpp"
#include "jfr/recorder/stringpool/jfrStringPoolWriter.hpp"
#include "jfr/utilities/jfrLinkedList.inline.hpp"
#include "jfr/utilities/jfrTypes.hpp"
#include "logging/log.hpp"
#include "runtime/atomic.hpp"
#include "runtime/mutexLocker.hpp"
#include "runtime/safepoint.hpp"
#include "runtime/thread.inline.hpp"
typedef JfrStringPool::Buffer* BufferPtr;
typedef JfrStringPool::BufferPtr BufferPtr;
static JfrStringPool* _instance = NULL;
static uint64_t store_generation = 0;
@ -48,6 +48,7 @@ inline void set_generation(uint64_t value, uint64_t* const dest) {
assert(dest != NULL, "invariant");
Atomic::release_store(dest, value);
}
static void increment_store_generation() {
const uint64_t current_serialized = Atomic::load_acquire(&serialized_generation);
const uint64_t current_stored = Atomic::load_acquire(&store_generation);
@ -88,14 +89,11 @@ void JfrStringPool::destroy() {
_instance = NULL;
}
JfrStringPool::JfrStringPool(JfrChunkWriter& cw) : _free_list_mspace(NULL), _lock(NULL), _chunkwriter(cw) {}
JfrStringPool::JfrStringPool(JfrChunkWriter& cw) : _mspace(NULL), _chunkwriter(cw) {}
JfrStringPool::~JfrStringPool() {
if (_free_list_mspace != NULL) {
delete _free_list_mspace;
}
if (_lock != NULL) {
delete _lock;
if (_mspace != NULL) {
delete _mspace;
}
}
@ -104,14 +102,9 @@ static const size_t string_pool_cache_count = 2;
static const size_t string_pool_buffer_size = 512 * K;
bool JfrStringPool::initialize() {
assert(_free_list_mspace == NULL, "invariant");
_free_list_mspace = new JfrStringPoolMspace(string_pool_buffer_size, unlimited_mspace_size, string_pool_cache_count, this);
if (_free_list_mspace == NULL || !_free_list_mspace->initialize()) {
return false;
}
assert(_lock == NULL, "invariant");
_lock = new Mutex(Monitor::leaf - 1, "Checkpoint mutex", Mutex::_allow_vm_block_flag, Monitor::_safepoint_check_never);
return _lock != NULL;
assert(_mspace == NULL, "invariant");
_mspace = create_mspace<JfrStringPoolMspace>(string_pool_buffer_size, unlimited_mspace_size, string_pool_cache_count, this);
return _mspace != NULL;
}
/*
@ -125,7 +118,11 @@ static void release(BufferPtr buffer, Thread* thread) {
assert(buffer->lease(), "invariant");
assert(buffer->acquired_by_self(), "invariant");
buffer->clear_lease();
buffer->release();
if (buffer->transient()) {
buffer->set_retired();
} else {
buffer->release();
}
}
BufferPtr JfrStringPool::flush(BufferPtr old, size_t used, size_t requested, Thread* thread) {
@ -137,7 +134,7 @@ BufferPtr JfrStringPool::flush(BufferPtr old, size_t used, size_t requested, Thr
return NULL;
}
// migration of in-flight information
BufferPtr const new_buffer = lease_buffer(thread, used + requested);
BufferPtr const new_buffer = lease(thread, used + requested);
if (new_buffer != NULL) {
migrate_outstanding_writes(old, new_buffer, used, requested);
}
@ -147,10 +144,10 @@ BufferPtr JfrStringPool::flush(BufferPtr old, size_t used, size_t requested, Thr
static const size_t lease_retry = 10;
BufferPtr JfrStringPool::lease_buffer(Thread* thread, size_t size /* 0 */) {
BufferPtr buffer = mspace_get_free_lease_with_retry(size, instance()._free_list_mspace, lease_retry, thread);
BufferPtr JfrStringPool::lease(Thread* thread, size_t size /* 0 */) {
BufferPtr buffer = mspace_get_free_lease_with_retry(size, instance()._mspace, lease_retry, thread);
if (buffer == NULL) {
buffer = mspace_allocate_transient_lease_to_free(size, instance()._free_list_mspace, thread);
buffer = mspace_allocate_transient_lease_to_full(size, instance()._mspace, thread);
}
assert(buffer->acquired_by_self(), "invariant");
assert(buffer->lease(), "invariant");
@ -210,18 +207,23 @@ typedef StringPoolOp<UnBufferedWriteToChunk> WriteOperation;
typedef StringPoolOp<StringPoolDiscarderStub> DiscardOperation;
typedef ExclusiveOp<WriteOperation> ExclusiveWriteOperation;
typedef ExclusiveOp<DiscardOperation> ExclusiveDiscardOperation;
typedef ReleaseOp<JfrStringPoolMspace> StringPoolReleaseOperation;
typedef CompositeOperation<ExclusiveWriteOperation, StringPoolReleaseOperation> StringPoolWriteOperation;
typedef CompositeOperation<ExclusiveDiscardOperation, StringPoolReleaseOperation> StringPoolDiscardOperation;
typedef ReleaseOp<JfrStringPoolMspace> StringPoolReleaseFreeOperation;
typedef ScavengingReleaseOp<JfrStringPoolMspace> StringPoolReleaseFullOperation;
typedef CompositeOperation<ExclusiveWriteOperation, StringPoolReleaseFreeOperation> StringPoolWriteFreeOperation;
typedef CompositeOperation<ExclusiveWriteOperation, StringPoolReleaseFullOperation> StringPoolWriteFullOperation;
typedef CompositeOperation<ExclusiveDiscardOperation, StringPoolReleaseFreeOperation> StringPoolDiscardFreeOperation;
typedef CompositeOperation<ExclusiveDiscardOperation, StringPoolReleaseFullOperation> StringPoolDiscardFullOperation;
size_t JfrStringPool::write() {
Thread* const thread = Thread::current();
WriteOperation wo(_chunkwriter, thread);
ExclusiveWriteOperation ewo(wo);
StringPoolReleaseOperation spro(_free_list_mspace, thread, false);
StringPoolWriteOperation spwo(&ewo, &spro);
assert(_free_list_mspace->is_full_empty(), "invariant");
process_free_list(spwo, _free_list_mspace);
StringPoolReleaseFreeOperation free_release_op(_mspace);
StringPoolWriteFreeOperation free_op(&ewo, &free_release_op);
process_free_list(free_op, _mspace);
StringPoolReleaseFullOperation full_release_op(_mspace);
StringPoolWriteFullOperation full_op(&ewo, &full_release_op);
process_full_list(full_op, _mspace);
return wo.processed();
}
@ -234,31 +236,18 @@ size_t JfrStringPool::clear() {
increment_serialized_generation();
DiscardOperation discard_operation;
ExclusiveDiscardOperation edo(discard_operation);
StringPoolReleaseOperation spro(_free_list_mspace, Thread::current(), false);
StringPoolDiscardOperation spdo(&edo, &spro);
assert(_free_list_mspace->is_full_empty(), "invariant");
process_free_list(spdo, _free_list_mspace);
StringPoolReleaseFreeOperation free_release_op(_mspace);
StringPoolDiscardFreeOperation free_op(&edo, &free_release_op);
process_free_list(free_op, _mspace);
StringPoolReleaseFullOperation full_release_op(_mspace);
StringPoolDiscardFullOperation full_op(&edo, &full_release_op);
process_full_list(full_op, _mspace);
return discard_operation.processed();
}
void JfrStringPool::register_full(BufferPtr t, Thread* thread) {
void JfrStringPool::register_full(BufferPtr buffer, Thread* thread) {
// nothing here at the moment
assert(t != NULL, "invariant");
assert(t->acquired_by(thread), "invariant");
assert(t->retired(), "invariant");
assert(buffer != NULL, "invariant");
assert(buffer->acquired_by(thread), "invariant");
assert(buffer->retired(), "invariant");
}
void JfrStringPool::lock() {
assert(!_lock->owned_by_self(), "invariant");
_lock->lock_without_safepoint_check();
}
void JfrStringPool::unlock() {
_lock->unlock();
}
#ifdef ASSERT
bool JfrStringPool::is_locked() const {
return _lock->owned_by_self();
}
#endif

27
src/hotspot/share/jfr/recorder/stringpool/jfrStringPool.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -29,12 +29,12 @@
#include "jfr/recorder/storage/jfrMemorySpace.hpp"
#include "jfr/recorder/storage/jfrMemorySpaceRetrieval.hpp"
#include "jfr/recorder/stringpool/jfrStringPoolBuffer.hpp"
#include "jfr/utilities/jfrLinkedList.hpp"
class JfrChunkWriter;
class JfrStringPool;
class Mutex;
typedef JfrMemorySpace<JfrStringPoolBuffer, JfrMspaceSequentialRetrieval, JfrStringPool> JfrStringPoolMspace;
typedef JfrMemorySpace<JfrStringPool, JfrMspaceRetrieval, JfrLinkedList<JfrStringPoolBuffer> > JfrStringPoolMspace;
//
// Although called JfrStringPool, a more succinct description would be
@ -48,21 +48,15 @@ class JfrStringPool : public JfrCHeapObj {
size_t write();
size_t write_at_safepoint();
size_t clear();
typedef JfrStringPoolMspace::Node Buffer;
typedef JfrStringPoolMspace::NodePtr BufferPtr;
typedef JfrStringPoolMspace::Type Buffer;
private:
JfrStringPoolMspace* _free_list_mspace;
Mutex* _lock;
JfrStringPoolMspace* _mspace;
JfrChunkWriter& _chunkwriter;
// mspace callback
void register_full(Buffer* t, Thread* thread);
void lock();
void unlock();
DEBUG_ONLY(bool is_locked() const;)
static Buffer* lease_buffer(Thread* thread, size_t size = 0);
static Buffer* flush(Buffer* old, size_t used, size_t requested, Thread* t);
static BufferPtr lease(Thread* thread, size_t size = 0);
static BufferPtr flush(BufferPtr old, size_t used, size_t requested, Thread* thread);
JfrStringPool(JfrChunkWriter& cw);
~JfrStringPool();
@ -73,11 +67,14 @@ class JfrStringPool : public JfrCHeapObj {
static void destroy();
static bool is_modified();
// mspace callback
void register_full(BufferPtr buffer, Thread* thread);
friend class JfrRecorder;
friend class JfrRecorderService;
friend class JfrStringPoolFlush;
friend class JfrStringPoolWriter;
template <typename, template <typename> class, typename>
template <typename, template <typename> class, typename, typename>
friend class JfrMemorySpace;
};

7
src/hotspot/share/jfr/recorder/stringpool/jfrStringPoolBuffer.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -41,6 +41,11 @@ class JfrStringPoolBuffer : public JfrBuffer {
void increment(uint64_t value);
void set_string_pos(uint64_t value);
void set_string_top(uint64_t value);
template <typename, typename>
friend class JfrLinkedList;
template <typename, typename>
friend class JfrConcurrentLinkedList;
};
#endif // SHARE_JFR_RECORDER_STRINGPOOL_JFRSTRINGPOOLBUFFER_HPP

8
src/hotspot/share/jfr/recorder/stringpool/jfrStringPoolWriter.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -28,11 +28,11 @@
#include "jfr/writers/jfrEventWriterHost.inline.hpp"
#include "jfr/writers/jfrMemoryWriterHost.inline.hpp"
JfrStringPoolFlush::JfrStringPoolFlush(Type* old, size_t used, size_t requested, Thread* t) :
_result(JfrStringPool::flush(old, used, requested, t)) {}
JfrStringPoolFlush::JfrStringPoolFlush(Type* old, size_t used, size_t requested, Thread* thread) :
_result(JfrStringPool::flush(old, used, requested, thread)) {}
JfrStringPoolWriter::JfrStringPoolWriter(Thread* thread) :
JfrStringPoolWriterBase(JfrStringPool::lease_buffer(thread), thread), _nof_strings(0) {}
JfrStringPoolWriterBase(JfrStringPool::lease(thread), thread), _nof_strings(0) {}
JfrStringPoolWriter::~JfrStringPoolWriter() {
assert(this->is_acquired(), "invariant");

24
src/hotspot/share/jfr/support/jfrFlush.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -31,8 +31,8 @@
#include "runtime/thread.inline.hpp"
#include "utilities/debug.hpp"
JfrFlush::JfrFlush(JfrStorage::Buffer* old, size_t used, size_t requested, Thread* t) :
_result(JfrStorage::flush(old, used, requested, true, t)) {
JfrFlush::JfrFlush(JfrStorage::BufferPtr old, size_t used, size_t requested, Thread* thread) :
_result(JfrStorage::flush(old, used, requested, true, thread)) {
}
template <typename T>
@ -61,24 +61,24 @@ bool jfr_has_stacktrace_enabled(JfrEventId id) {
return JfrEventSetting::has_stacktrace(id);
}
void jfr_conditional_flush(JfrEventId id, size_t size, Thread* t) {
if (t->jfr_thread_local()->has_native_buffer()) {
JfrStorage::Buffer* const buffer = t->jfr_thread_local()->native_buffer();
void jfr_conditional_flush(JfrEventId id, size_t size, Thread* thread) {
if (thread->jfr_thread_local()->has_native_buffer()) {
JfrStorage::BufferPtr buffer = thread->jfr_thread_local()->native_buffer();
if (LessThanSize<JfrStorage::Buffer>::evaluate(buffer, size)) {
JfrFlush f(buffer, 0, 0, t);
JfrFlush f(buffer, 0, 0, thread);
}
}
}
bool jfr_save_stacktrace(Thread* t) {
JfrThreadLocal* const tl = t->jfr_thread_local();
bool jfr_save_stacktrace(Thread* thread) {
JfrThreadLocal* const tl = thread->jfr_thread_local();
if (tl->has_cached_stack_trace()) {
return false; // no ownership
}
tl->set_cached_stack_trace_id(JfrStackTraceRepository::record(t));
tl->set_cached_stack_trace_id(JfrStackTraceRepository::record(thread));
return true;
}
void jfr_clear_stacktrace(Thread* t) {
t->jfr_thread_local()->clear_cached_stack_trace();
void jfr_clear_stacktrace(Thread* thread) {
thread->jfr_thread_local()->clear_cached_stack_trace();
}

98
src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.hpp Normal file
View File

@ -0,0 +1,98 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_HPP
#define SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_HPP
#include "jfr/utilities/jfrAllocation.hpp"
/*
* This implementation is a derivation from Harris
* https://www.cl.cam.ac.uk/research/srg/netos/papers/2001-caslists.pdf
*
* A concurrent LIFO structure can be built using the pair:
*
* insert_head() and remove()
*
* The LIFO algorithm is non-blocking, more specifically wait-free.
* When combined with a system for safe memory reclamation, where a thread will require
* to know if other threads are possibly reading the memory that is to be reclaimed (more below),
* a potential wait point is introduced, so technically, we are no longer wait-free.
* The combination is still lock-free, but since it is no longer pure non-blocking,
* we instead say the solution is concurrent.
*
* It is also possible to build a FIFO structure using the pair:
*
* insert_tail() and remove()
*
* To allow FIFO, the solution extends support to mark, or reserve a node, not only as part of deletions
* as with the LIFO case, but also, to enable tail insertions.
*
* Compared to the LIFO algorithm, the FIFO algorithm is not non-blocking, because inserts to the tail block,
* making it not lock-free. remove() is lock-free up until the last node in the list. In practice, the FIFO
* solution can be used in certain ways that very closely approximate non-blocking, for example, situations
* involving a single producer and multiple consumers.
*
* Although the FIFO algorithm is not non-blocking, it includes an optimization for remove() that is attractive:
* In the LIFO case, a slow path taken as the result of a failed excision would have to re-traverse the list
* to find the updated adjacent node pair for the already marked node. However, that node might already have
* been excised by some other thread, letting the thread potentially traverse the entire list just to discover
* it is no longer present (not an issue if the list is ordered by a key, then traversal is only to node >= key).
* In the FIFO case, premised on the invariant that inserts only come in from the tail, it is possible to prove
* a failed cas not to be the result of a new node inserted as with the LIFO case. With FIFO, there is only a single
* failure mode, i.e. some other thread excised the node already. Therefore, in the FIFO case, we skip the slow-path search pass.
*
* We say that the FIFO solution is "mostly" concurrent, in certain situations.
*
* Safe memory reclamation is based on a reference tracking scheme based on versions, implemented using JfrVersion.
* An access to the list is "version controlled", with clients checking out the latest version of the list.
* Destructive modifications made by clients, i.e. deletions, are committed to describe new versions of the list.
* Before reclamation, a client inspects the versioning system to ensure checkouts for versions strictly
* less than the version of the modification have all been relinquished. See utilities/JfrVersion.hpp.
*
* Insertions can only take place from one end of the list, head or tail, exclusively.
* Specializations, a.k.a clients, must ensure this requirement.
*/
template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy = JfrCHeapObj>
class JfrConcurrentLinkedListHost : public AllocPolicy {
private:
Client* _client;
typedef typename Client::Node Node;
typedef Node* NodePtr;
typedef const Node* ConstNodePtr;
typedef typename Client::VersionSystem::Type VersionType;
typedef typename Client::VersionSystem::Handle VersionHandle;
public:
JfrConcurrentLinkedListHost(Client* client);
bool initialize();
void insert_head(NodePtr node, NodePtr head, ConstNodePtr tail) const;
void insert_tail(NodePtr node, NodePtr head, NodePtr last, ConstNodePtr tail) const;
NodePtr remove(NodePtr head, ConstNodePtr tail, NodePtr last = NULL, bool insert_is_head = true);
template <typename Callback>
void iterate(NodePtr head, ConstNodePtr tail, Callback& cb);
bool in_list(ConstNodePtr node, NodePtr head, ConstNodePtr tail) const;
};
#endif // SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_HPP

299
src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp Normal file
View File

@ -0,0 +1,299 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_INLINE_HPP
#define SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_INLINE_HPP
#include "jfr/utilities/jfrConcurrentLinkedListHost.hpp"
#include "jfr/utilities/jfrRelation.hpp"
#include "jfr/utilities/jfrTypes.hpp"
#include "runtime/atomic.hpp"
#include "runtime/os.inline.hpp"
#include "utilities/globalDefinitions.hpp"
/*
* The removal marker (i.e. the excision bit) is represented by '( )' as part of state description comments:
* node --> next becomes (node) --> next, when node is logically deleted.
*/
template <typename Node>
inline Node* mark_for_removal(Node* node) {
assert(node != NULL, "invariant");
const Node* next = node->_next;
assert(next != NULL, "invariant");
Node* const unmasked_next = unmask(next);
return next == unmasked_next && cas(&node->_next, unmasked_next, set_excision_bit(unmasked_next)) ? unmasked_next : NULL;
}
/*
* The insertion marker (i.e. the insertion bit) is represented by '[ ]' as part of state description comments:
* "node --> next" becomes "[node} --> next", in an attempt to convey node as being exlusively reserved.
*/
template <typename Node>
inline bool mark_for_insertion(Node* node, const Node* tail) {
assert(node != NULL, "invariant");
return node->_next == tail && cas(&node->_next, const_cast<Node*>(tail), set_insertion_bit(tail));
}
/*
* Find a predecessor and successor node pair where successor covers predecessor (adjacency).
*/
template <typename Node, typename VersionHandle, template <typename> class SearchPolicy>
Node* find_adjacent(Node* head, const Node* tail, Node** predecessor, VersionHandle& version_handle, SearchPolicy<Node>& predicate) {
assert(head != NULL, "invariant");
assert(tail != NULL, "invariant");
assert(head != tail, "invariant");
while (true) {
Node* predecessor_next;
Node* current = head;
version_handle.checkout();
assert(version_handle.is_tracked(), "invariant");
Node* next = Atomic::load_acquire(&current->_next);
do {
assert(next != NULL, "invariant");
Node* const unmasked_next = unmask(next);
// 1A: Locate the first node to keep as predecessor.
if (!is_marked_for_removal(next)) {
*predecessor = current;
predecessor_next = unmasked_next;
}
// 1B: Locate the next node to keep as successor.
current = unmasked_next;
if (current == tail) break;
next = current->_next;
} while (predicate(current, next));
// current represents the successor node from here on out.
// 2: Check predecessor and successor node pair for adjacency.
if (predecessor_next == current) {
// Invariant: predecessor --> successor
return current;
}
// 3: Successor does not (yet) cover predecessor.
// Invariant: predecessor --> (logically excised nodes) --> successor
// Physically excise one or more logically excised nodes in-between.
if (cas(&(*predecessor)->_next, predecessor_next, current)) {
// Invariant: predecessor --> successor
return current;
}
}
}
template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::JfrConcurrentLinkedListHost(Client* client) : _client(client) {}
template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
bool JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::initialize() {
return true;
}
template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
void JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::insert_head(typename Client::Node* node,
typename Client::Node* head,
const typename Client::Node* tail) const {
Node* predecessor;
Node* successor;
HeadNode<Node> predicate(node);
VersionHandle version_handle = _client->get_version_handle();
while (true) {
// Find an adjacent predecessor and successor node pair.
successor = find_adjacent<Node, VersionHandle, HeadNode>(head, tail, &predecessor, version_handle, predicate);
assert(version_handle.is_tracked(), "invariant");
// Invariant (adjacency): predecessor --> successor
// Invariant (optional: key-based total order): predecessor->key() < key && key <= successor->key().
// We can now attempt to insert the new node in-between.
node->_next = successor;
if (cas(&predecessor->_next, successor, node)) {
// Invariant: predecessor --> node --> successor
// An insert to head is a benign modification and will not need to be committed to the version control system.
return;
}
}
}
template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
void JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::insert_tail(typename Client::Node* node,
typename Client::Node* head,
typename Client::Node* last,
const typename Client::Node* tail) const {
assert(node != NULL, "invariant");
assert(head != NULL, "invariant");
assert(last != NULL, "invarinat");
assert(tail != NULL, "invariant");
// Mark the new node to be inserted with the insertion marker already.
node->_next = set_insertion_bit(const_cast<NodePtr>(tail));
// Invariant: [node]--> tail
assert(is_marked_for_insertion(node->_next), "invariant");
NodePtr predecessor;
LastNode<Node> predicate;
VersionHandle version_handle = _client->get_version_handle();
while (true) {
// Find an adjacent predecessor and successor node pair, where the successor == tail
const NodePtr successor = find_adjacent<Node, VersionHandle, LastNode>(last, tail, &predecessor, version_handle, predicate);
assert(version_handle.is_tracked(), "invariant");
assert(successor == tail, "invariant");
// Invariant: predecessor --> successor
// We first attempt to mark the predecessor node to signal our intent of performing an insertion.
if (mark_for_insertion(predecessor, tail)) {
break;
}
}
// Predecessor node is claimed for insertion.
// Invariant: [predecessor] --> tail
assert(is_marked_for_insertion(predecessor->_next), "invariant");
assert(predecessor != head, "invariant");
if (Atomic::load_acquire(&last->_next) == predecessor) {
/* Even after we store the new node into the last->_next field, there is no race
because it is also marked with the insertion bit. */
last->_next = node;
// Invariant: last --> [node] --> tail
OrderAccess::storestore();
// Perform the link with the predecessor node, which by this store becomes visible for removal.
predecessor->_next = node;
// Invariant: predecessor --> [node] --> tail
} else {
assert(last == predecessor, "invariant");
last->_next = node;
// Invariant: last --> [node] --> tail
OrderAccess::storestore();
/* This implies the list is logically empty from the removal perspective.
cas is not needed here because inserts must not come in from the head side
concurrently with inserts from tail which are currently blocked by us.
Invariant (logical): head --> tail. */
head->_next = node;
// Invariant: head --> [node] --> tail
}
version_handle.release(); // release_store_fence
// Publish the inserted node by removing the insertion marker.
node->_next = const_cast<NodePtr>(tail);
// Invariant: last --> node --> tail (possibly also head --> node --> tail)
}
template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
typename Client::Node* JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::remove(typename Client::Node* head,
const typename Client::Node* tail,
typename Client::Node* last /* NULL */,
bool insert_is_head /* true */) {
assert(head != NULL, "invariant");
assert(tail != NULL, "invariant");
assert(head != tail, "invariant");
NodePtr predecessor;
NodePtr successor;
NodePtr successor_next;
SearchPolicy<Node> predicate;
VersionHandle version_handle = _client->get_version_handle();
while (true) {
// Find an adjacent predecessor and successor node pair.
successor = find_adjacent<Node, VersionHandle, SearchPolicy>(head, tail, &predecessor, version_handle, predicate);
assert(version_handle.is_tracked(), "invariant");
if (successor == tail) {
return NULL;
}
// Invariant: predecessor --> successor
// Invariant (optional: key-based total order): predecessor->key() < key && key <= successor->key()
// It is the successor node that is to be removed.
// We first attempt to reserve (logically excise) the successor node.
successor_next = mark_for_removal(successor);
if (successor_next != NULL) {
break;
}
}
// Invariant: predecessor --> (successor) --> successor_next
// Successor node now logically excised.
assert(is_marked_for_removal(successor->_next), "invariant");
// Now attempt to physically excise the successor node.
// If the cas fails, we can optimize for the slow path if we know we are not performing
// insertions from the head. Then a failed cas results not from new a node being inserted,
// but only because another thread excised us already.
if (!cas(&predecessor->_next, successor, successor_next) && insert_is_head) {
// Physically excise using slow path, can be completed asynchronously by other threads.
Identity<Node> excise(successor);
find_adjacent<Node, VersionHandle, Identity>(head, tail, &predecessor, version_handle, excise);
assert(version_handle.is_tracked(), "invariant");
}
if (last != NULL && Atomic::load_acquire(&last->_next) == successor) {
guarantee(!insert_is_head, "invariant");
guarantee(successor_next == tail, "invariant");
LastNode<Node> excise;
find_adjacent<Node, VersionHandle, LastNode>(last, tail, &predecessor, version_handle, excise);
// Invariant: successor excised from last list
}
// Increment the current version so we can track when other threads have seen this update.
VersionType version = version_handle.increment();
version_handle.release(); // release_store_fence
// Rendezvous with checkouts for versions less than this version.
version_handle.await(version);
// At this point we know there can be no references onto the excised node. It is safe, enjoy it.
return successor;
}
template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
bool JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::in_list(const typename Client::Node* node,
typename Client::Node* head,
const typename Client::Node* tail) const {
assert(head != NULL, "invariant");
assert(tail != NULL, "invariant");
assert(head != tail, "invariant");
VersionHandle version_handle = _client->get_version_handle();
const Node* current = head;
version_handle.checkout();
assert(version_handle.is_tracked(), "invariant");
const Node* next = Atomic::load_acquire(&current->_next);
while (true) {
if (!is_marked_for_removal(next)) {
if (current == node) {
return true;
}
}
current = unmask(next);
if (current == tail) break;
next = current->_next;
}
return false;
}
template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
template <typename Callback>
inline void JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::iterate(typename Client::Node* head,
const typename Client::Node* tail,
Callback& cb) {
assert(head != NULL, "invariant");
assert(tail != NULL, "invariant");
assert(head != tail, "invariant");
VersionHandle version_handle = _client->get_version_handle();
NodePtr current = head;
version_handle.checkout();
assert(version_handle.is_tracked(), "invariant");
NodePtr next = Atomic::load_acquire(&current->_next);
while (true) {
if (!is_marked_for_removal(next)) {
if (!cb.process(current)) {
return;
}
}
current = unmask(next);
if (current == tail) break;
next = current->_next;
}
}
#endif // SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_INLINE_HPP

69
src/hotspot/share/jfr/utilities/jfrConcurrentQueue.hpp Normal file
View File

@ -0,0 +1,69 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_HPP
#define SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_HPP
#include "jfr/utilities/jfrAllocation.hpp"
#include "jfr/utilities/jfrConcurrentLinkedListHost.hpp"
#include "jfr/utilities/jfrRelation.hpp"
#include "jfr/utilities/jfrVersionSystem.hpp"
/*
* This is a thread-safe FIFO structure.
* Although not non-blocking, for certain scenarios,
* it can act as a close approximation, "mostly" concurrent.
* For a more detailed description of its properties,
* please see JfrConcurrentLinkedListHost.hpp.
*/
template <typename NodeType, typename AllocPolicy = JfrCHeapObj>
class JfrConcurrentQueue : public AllocPolicy {
public:
typedef NodeType Node;
typedef NodeType* NodePtr;
typedef const NodeType* ConstNodePtr;
typedef JfrVersionSystem VersionSystem;
JfrConcurrentQueue();
bool initialize();
bool is_empty() const;
bool is_nonempty() const;
void add(NodePtr node);
NodePtr remove();
template <typename Callback>
void iterate(Callback& cb);
bool in_list(const Node* node) const;
private:
JfrConcurrentLinkedListHost<JfrConcurrentQueue<Node, AllocPolicy>, HeadNode>* _list;
Node _head;
Node _last;
const Node _tail;
JfrVersionSystem _version_system;
// callback for JfrConcurrentLinkedListHost
typename VersionSystem::Handle get_version_handle();
template <typename, template <typename> class, typename>
friend class JfrConcurrentLinkedListHost;
};
#endif // SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_HPP

82
src/hotspot/share/jfr/utilities/jfrConcurrentQueue.inline.hpp Normal file
View File

@ -0,0 +1,82 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_INLINE_HPP
#define SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_INLINE_HPP
#include "jfr/utilities/jfrConcurrentQueue.hpp"
#include "jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
#include "jfr/utilities/jfrVersionSystem.inline.hpp"
template <typename NodeType, typename AllocPolicy>
JfrConcurrentQueue<NodeType, AllocPolicy>::JfrConcurrentQueue() : _list(NULL), _head(), _last(), _tail(), _version_system() {
_head._next = const_cast<NodePtr>(&_tail);
_last._next = const_cast<NodePtr>(&_tail);
}
template <typename NodeType, typename AllocPolicy>
bool JfrConcurrentQueue<NodeType, AllocPolicy>::initialize() {
assert(_list == NULL, "invariant");
_list = new JfrConcurrentLinkedListHost<JfrConcurrentQueue<NodeType, AllocPolicy>, HeadNode, AllocPolicy>(this);
return _list != NULL && _list->initialize();
}
template <typename NodeType, typename AllocPolicy>
inline bool JfrConcurrentQueue<NodeType, AllocPolicy>::is_empty() const {
return Atomic::load_acquire(&_head._next) == &_tail;
}
template <typename NodeType, typename AllocPolicy>
inline bool JfrConcurrentQueue<NodeType, AllocPolicy>::is_nonempty() const {
return !is_empty();
}
template <typename NodeType, typename AllocPolicy>
void JfrConcurrentQueue<NodeType, AllocPolicy>::add(typename JfrConcurrentQueue<NodeType, AllocPolicy>::NodePtr node) {
_list->insert_tail(node, &_head, &_last, &_tail);
}
template <typename NodeType, typename AllocPolicy>
typename JfrConcurrentQueue<NodeType, AllocPolicy>::NodePtr JfrConcurrentQueue<NodeType, AllocPolicy>::remove() {
return _list->remove(&_head, &_tail, &_last, false);
}
template <typename NodeType, typename AllocPolicy>
template <typename Callback>
void JfrConcurrentQueue<NodeType, AllocPolicy>::iterate(Callback& cb) {
_list->iterate(&_head, &_tail, cb);
}
template <typename NodeType, typename AllocPolicy>
inline JfrVersionSystem::Handle JfrConcurrentQueue<NodeType, AllocPolicy>::get_version_handle() {
return _version_system.get_handle();
}
template <typename NodeType, typename AllocPolicy>
bool JfrConcurrentQueue<NodeType, AllocPolicy>::in_list(const NodeType* node) const {
assert(node != NULL, "invariant");
return _list->in_list(node, const_cast<NodePtr>(&_head), &_tail);
}
#endif // SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_INLINE_HPP

96
src/hotspot/share/jfr/utilities/jfrIterator.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -27,16 +27,40 @@
#include "memory/allocation.hpp"
enum jfr_iter_direction {
forward = 1,
backward
template <typename List>
class StopOnNullCondition {
typedef typename List::Node Node;
private:
List& _list;
mutable Node* _node;
public:
StopOnNullCondition(List& list) : _list(list), _node(list.head()) {}
bool has_next() const {
return _node != NULL;
}
Node* next() const {
assert(_node != NULL, "invariant");
Node* temp = _node;
_node = (Node*)_node->_next;
return temp;
}
};
template <typename Node>
class StopOnNullCondition : public AllStatic {
template <typename List>
class StopOnNullConditionRemoval {
typedef typename List::Node Node;
private:
List& _list;
mutable Node* _node;
public:
static bool has_next(const Node* node) {
return node != NULL;
StopOnNullConditionRemoval(List& list) : _list(list), _node(NULL) {}
bool has_next() const {
_node = _list.remove();
return _node != NULL;
}
Node* next() const {
assert(_node != NULL, "invariant");
return _node;
}
};
@ -44,64 +68,52 @@ template <typename List, template <typename> class ContinuationPredicate>
class Navigator {
public:
typedef typename List::Node Node;
typedef jfr_iter_direction Direction;
Navigator(List& list, Direction direction) :
_list(list), _node(direction == forward ? list.head() : list.tail()), _direction(direction) {}
Navigator(List& list) : _continuation(list) {}
bool has_next() const {
return ContinuationPredicate<Node>::has_next(_node);
return _continuation.has_next();
}
bool direction_forward() const {
return _direction == forward;
}
Node* next() const {
assert(_node != NULL, "invariant");
Node* temp = _node;
_node = direction_forward() ? (Node*)_node->next() : (Node*)_node->prev();
return temp;
return _continuation.next();
}
void set_direction(Direction direction) {
_direction = direction;
}
void reset(Direction direction) {
set_direction(direction);
_node = direction_forward() ? _list.head() : _list.tail();
}
private:
List& _list;
ContinuationPredicate<List> _continuation;
mutable Node* _node;
Direction _direction;
};
template <typename List>
class NavigatorStopOnNull : public Navigator<List, StopOnNullCondition> {
public:
NavigatorStopOnNull(List& list, jfr_iter_direction direction = forward) : Navigator<List, StopOnNullCondition>(list, direction) {}
NavigatorStopOnNull(List& list) : Navigator<List, StopOnNullCondition>(list) {}
};
template <typename List>
class NavigatorStopOnNullRemoval : public Navigator<List, StopOnNullConditionRemoval> {
public:
NavigatorStopOnNullRemoval(List& list) : Navigator<List, StopOnNullConditionRemoval>(list) {}
};
template<typename List, template <typename> class Navigator, typename AP = StackObj>
class IteratorHost : public AP {
private:
Navigator<List> _navigator;
public:
typedef typename List::Node Node;
typedef jfr_iter_direction Direction;
IteratorHost(List& list, Direction direction = forward) : AP(), _navigator(list, direction) {}
void reset(Direction direction = forward) { _navigator.reset(direction); }
typedef typename List::NodePtr NodePtr;
IteratorHost(List& list) : AP(), _navigator(list) {}
void reset() { _navigator.reset(); }
bool has_next() const { return _navigator.has_next(); }
Node* next() const { return _navigator.next(); }
void set_direction(Direction direction) { _navigator.set_direction(direction); }
NodePtr next() const { return _navigator.next(); }
};
template<typename List, typename AP = StackObj>
class StopOnNullIterator : public IteratorHost<List, NavigatorStopOnNull, AP> {
public:
StopOnNullIterator(List& list, jfr_iter_direction direction = forward) : IteratorHost<List, NavigatorStopOnNull, AP>(list, direction) {}
StopOnNullIterator(List& list) : IteratorHost<List, NavigatorStopOnNull, AP>(list) {}
};
template<typename List, typename AP = StackObj>
class StopOnNullIteratorRemoval : public IteratorHost<List, NavigatorStopOnNullRemoval, AP> {
public:
StopOnNullIteratorRemoval(List& list) : IteratorHost<List, NavigatorStopOnNullRemoval, AP>(list) {}
};
#endif // SHARE_JFR_UTILITIES_JFRITERATOR_HPP

56
src/hotspot/share/jfr/utilities/jfrLinkedList.hpp Normal file
View File

@ -0,0 +1,56 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_JFR_UTILITIES_JFRLINKEDLIST_HPP
#define SHARE_JFR_UTILITIES_JFRLINKEDLIST_HPP
#include "jfr/utilities/jfrAllocation.hpp"
/*
* This linked-list is thread-safe only for add,
* not for remove, iterate, excise and in_list.
* For multiple producers, single consumer.
*/
template <typename NodeType, typename AllocPolicy = JfrCHeapObj>
class JfrLinkedList : public AllocPolicy {
public:
typedef NodeType Node;
typedef NodeType* NodePtr;
JfrLinkedList();
bool initialize();
bool is_empty() const;
bool is_nonempty() const;
void add(NodePtr node);
NodePtr remove();
template <typename Callback>
void iterate(Callback& cb);
NodePtr head() const;
NodePtr excise(NodePtr prev, NodePtr node);
bool in_list(const NodeType* node) const;
private:
NodePtr _head;
};
#endif // SHARE_JFR_UTILITIES_JFRLINKEDLIST_HPP

120
src/hotspot/share/jfr/utilities/jfrLinkedList.inline.hpp Normal file
View File

@ -0,0 +1,120 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_JFR_UTILITIES_JFRLINKEDLIST_INLINE_HPP
#define SHARE_JFR_UTILITIES_JFRLINKEDLIST_INLINE_HPP
#include "jfr/utilities/jfrLinkedList.hpp"
#include "runtime/atomic.hpp"
template <typename NodeType, typename AllocPolicy>
JfrLinkedList<NodeType, AllocPolicy>::JfrLinkedList() : _head(NULL) {}
template <typename NodeType, typename AllocPolicy>
bool JfrLinkedList<NodeType, AllocPolicy>::initialize() {
return true;
}
template <typename NodeType, typename AllocPolicy>
inline NodeType* JfrLinkedList<NodeType, AllocPolicy>::head() const {
return (NodeType*)Atomic::load_acquire(&_head);
}
template <typename NodeType, typename AllocPolicy>
inline bool JfrLinkedList<NodeType, AllocPolicy>::is_empty() const {
return NULL == head();
}
template <typename NodeType, typename AllocPolicy>
inline bool JfrLinkedList<NodeType, AllocPolicy>::is_nonempty() const {
return !is_empty();
}
template <typename NodeType, typename AllocPolicy>
inline void JfrLinkedList<NodeType, AllocPolicy>::add(NodeType* node) {
assert(node != NULL, "invariant");
NodePtr next;
do {
next = head();
node->_next = next;
} while (Atomic::cmpxchg(&_head, next, node) != next);
}
template <typename NodeType, typename AllocPolicy>
inline NodeType* JfrLinkedList<NodeType, AllocPolicy>::remove() {
NodePtr node;
NodePtr next;
do {
node = head();
if (node == NULL) break;
next = (NodePtr)node->_next;
} while (Atomic::cmpxchg(&_head, node, next) != node);
return node;
}
template <typename NodeType, typename AllocPolicy>
template <typename Callback>
void JfrLinkedList<NodeType, AllocPolicy>::iterate(Callback& cb) {
NodePtr current = head();
while (current != NULL) {
NodePtr next = (NodePtr)current->_next;
if (!cb.process(current)) {
return;
}
current = next;
}
}
template <typename NodeType, typename AllocPolicy>
NodeType* JfrLinkedList<NodeType, AllocPolicy>::excise(NodeType* prev, NodeType* node) {
NodePtr next = (NodePtr)node->_next;
if (prev == NULL) {
prev = Atomic::cmpxchg(&_head, node, next);
if (prev == node) {
return NULL;
}
}
assert(prev != NULL, "invariant");
while (prev->_next != node) {
prev = (NodePtr)prev->_next;
}
assert(prev->_next == node, "invariant");
prev->_next = next;
return prev;
}
template <typename NodeType, typename AllocPolicy>
bool JfrLinkedList<NodeType, AllocPolicy>::in_list(const NodeType* node) const {
assert(node != NULL, "invariant");
const NodeType* current = head();
while (current != NULL) {
if (current == node) {
return true;
}
current = (NodeType*)current->_next;
}
return false;
}
#endif // SHARE_JFR_UTILITIES_JFRLINKEDLIST_INLINE_HPP

114
src/hotspot/share/jfr/utilities/jfrNode.hpp Normal file
View File

@ -0,0 +1,114 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_JFR_UTILITIES_JFRNODE_HPP
#define SHARE_JFR_UTILITIES_JFRNODE_HPP
#include "jfr/utilities/jfrTypes.hpp"
#include "memory/allocation.hpp"
#include "runtime/atomic.hpp"
const uint64_t JFR_NODE_LOGICAL_EXCISION_BIT = 1;
const uint64_t JFR_NODE_LOGICAL_INSERTION_BIT = 2;
const uint64_t JFR_NODE_MASK = ~(JFR_NODE_LOGICAL_INSERTION_BIT | JFR_NODE_LOGICAL_EXCISION_BIT);
template <typename Node>
inline bool cas(Node** address, Node* current, Node* exchange) {
return Atomic::cmpxchg(address, current, exchange) == current;
}
template <typename Node>
inline bool is_marked_for_removal(const Node* ptr) {
return ((uint64_t)ptr & JFR_NODE_LOGICAL_EXCISION_BIT) == JFR_NODE_LOGICAL_EXCISION_BIT;
}
template <typename Node>
inline bool is_marked_for_insertion(const Node* ptr) {
return ((uint64_t)ptr & JFR_NODE_LOGICAL_INSERTION_BIT) == JFR_NODE_LOGICAL_INSERTION_BIT;
}
template <typename Node>
inline Node* set_excision_bit(const Node* ptr) {
return (Node*)(((uint64_t)ptr) | JFR_NODE_LOGICAL_EXCISION_BIT);
}
template <typename Node>
inline Node* set_insertion_bit(const Node* ptr) {
return (Node*)(((uint64_t)ptr) | JFR_NODE_LOGICAL_INSERTION_BIT);
}
template <typename Node>
inline Node* unmask(const Node* ptr) {
return (Node*)(((uint64_t)ptr) & JFR_NODE_MASK);
}
template <typename Derived, typename Version = traceid>
class JfrLinkedNode : public ResourceObj {
public:
typedef Version VersionType;
Derived* _next;
JfrLinkedNode() : _next(NULL) {}
JfrLinkedNode(JfrLinkedNode<Derived, VersionType>* next) : _next(next) {}
};
template <typename V>
class JfrKeyIsThisNode : public JfrLinkedNode<JfrKeyIsThisNode<V> > {
private:
V _value;
public:
typedef V Value;
typedef const JfrKeyIsThisNode<V>* Key;
JfrKeyIsThisNode(const Value value = NULL) : JfrLinkedNode<JfrKeyIsThisNode<V> >(), _value(value) {}
Key key() const { return this; }
Value value() const { return _value; }
void set_value(Value value) { _value = value; }
};
template <typename V>
class JfrValueNode : public JfrLinkedNode<JfrValueNode<V> > {
private:
V _value;
public:
typedef V Value;
typedef Value Key;
JfrValueNode(const Value value = NULL) : JfrLinkedNode<JfrValueNode<V> >(), _value(value) {}
Key key() const { return value(); }
Value value() const { return _value; }
void set_value(Value value) { _value = value; }
};
template <typename V>
class JfrKeyIsFreeSizeNode : public JfrLinkedNode<JfrKeyIsFreeSizeNode<V> > {
private:
V _value;
public:
typedef V Value;
typedef size_t Key;
JfrKeyIsFreeSizeNode(const Value value = NULL) : JfrLinkedNode<JfrKeyIsFreeSizeNode<V> >(), _value(value) {}
Key key() const { return value()->free_size(); }
Value value() const { return _value; }
void set_value(Value value) { _value = value; }
};
#endif // SHARE_JFR_UTILITIES_JFRNODE_HPP

95
src/hotspot/share/jfr/utilities/jfrRelation.hpp Normal file
View File

@ -0,0 +1,95 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_JFR_UTILITIES_JFRRELATION_HPP
#define SHARE_JFR_UTILITIES_JFRRELATION_HPP
#include "jfr/utilities/jfrNode.hpp"
template <typename Node>
class LessThan {
private:
typename Node::Key _key;
public:
LessThan(typename Node::Key key) : _key(key) {}
bool operator()(const Node* node) { return node->key() < _key; }
};
template <typename Node>
class GreaterThan {
private:
typename Node::Key _key;
public:
GreaterThan(typename Node::Key key) : _key(key) {}
bool operator()(const Node* node) { return node->key() > _key; }
};
/*
* Using a contradiction as a search predicate amounts
* to using the physical order of the list (key is neglected).
* [LessThan relation -> Ascending order ] -> the minimal element.
* [GreaterThan relation -> Descending order] -> the maximal element.
*/
template <typename Node>
class HeadNode {
public:
HeadNode(const Node* node = NULL) {}
bool operator()(const Node* current, const Node* next) {
return is_marked_for_removal(next);
}
};
/*
* Using a tautology as a search predicate amounts
* to using the physical store order of the list (key is neglected).
* [LessThan relation -> Ascending order ] -> the maximal element.
* [GreaterThan relation -> Descending order] -> the minimal element.
*/
template <typename Node>
class LastNode {
public:
LastNode(const Node* node = NULL) {}
bool operator()(const Node* current, const Node* next ) {
return true;
}
};
template <typename Node>
class Identity {
private:
const Node* _target;
bool _found;
public:
Identity(const Node* node = NULL) : _target(node), _found(false) {}
bool operator()(const Node* current, const Node* next) {
assert(current != NULL, "invariant");
assert(next != NULL, "invariant");
if (!_found && current == _target) {
_found = true;
}
return is_marked_for_removal(next) || !_found;
}
};
#endif // SHARE_JFR_UTILITIES_JFRRELATION_HPP

122
src/hotspot/share/jfr/utilities/jfrVersionSystem.hpp Normal file
View File

@ -0,0 +1,122 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_JFR_UTILITIES_JFRVERSIONSYSTEM_HPP
#define SHARE_JFR_UTILITIES_JFRVERSIONSYSTEM_HPP
/*
* A lock-free data structure usually require support for tracking references
* in the service of Safe Memory Reclamation (SMR). JfrVersionSystem provides local,
* compared to global, reference tracking for an assoicated data structure.
*
* A client, before accessing a structure, will perform a "checkout" from the JfrVersionSystem.
* This checkout is associated with the current, or latest, version, analogous to the "tip" in a version control system.
* When a client is done it releases its checkout, by which the JfrVersionSystem is notified that the client
* is no longer an active user of the associated structure.
*
* If a client performs a modification, it will register this with the JfrVersionSystem, by means of incrementing the current version.
*
* To guarantee safe memory reclamation, say before attempting a delete, a client will use the JfrVersionSystem to
* check for potential active uses, i.e. checkouts, with versions earlier than the version associated with a specific modification.
*
* Let's exemplify this with the removal of a node from a linked-list:
*
* 1. Before accessing the list, the client will check out the latest version from the JfrVersionSystem. This access is now tracked.
* 2. The client finds a node it wants to use, and excises the node from the list.
* 3. This excision is a modification so the client will increment the current version.
* 4. Before it can start to use proper the node just excised, the client must ensure no possible references exist.
* 5. To do this, the client inspects the JfrVersionSystem to possibly await the release of existing checkouts,
* i.e. for versions less than the version associated with the modification.
* 6. On return, the client is guaranteed exclusive access to the excised node.
*
* Tracking the version of a structure is conceptually similar to tracking a representative pointer using Hazard Pointers,
* or by using a global counter or some kind of ticket system.
*
* The implementation was inspired by Andrei Alexandrescu and Maged Michael, Lock-Free Data Structures with Hazard Pointers
* https ://www.drdobbs.com/lock-free-data-structures-with-hazard-po/184401890
*/
#include "jfr/utilities/jfrAllocation.hpp"
#include "jfr/utilities/jfrTypes.hpp"
#include "memory/padded.hpp"
class JfrVersionSystem : public JfrCHeapObj {
public:
typedef traceid Type;
private:
class Node : public JfrCHeapObj {
public:
Node* _next;
Type _version;
bool _live;
Node();
Type version() const;
void set(Type version);
};
typedef Node* NodePtr;
public:
class Handle {
private:
JfrVersionSystem* _system;
NodePtr _node;
Handle(JfrVersionSystem* system);
public:
Handle();
~Handle();
void checkout();
void release();
Type increment();
void await(Type version);
DEBUG_ONLY(bool is_tracked() const;)
friend class JfrVersionSystem;
};
JfrVersionSystem();
~JfrVersionSystem();
void reset();
// to access the versioning system
Handle get_handle();
Handle checkout_handle();
private:
struct PaddedTip {
DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, 0);
volatile Type _value;
DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile Type));
};
PaddedTip _tip;
NodePtr _head;
NodePtr acquire();
void release(NodePtr node);
void await(Type version);
Type tip() const;
Type increment();
NodePtr synchronize_with(Type version, NodePtr last) const;
debug_only(bool is_registered(Type version) const;)
friend class Handle;
};
#endif // SHARE_JFR_UTILITIES_JFRVERSIONSYSTEM_HPP

185
src/hotspot/share/jfr/utilities/jfrVersionSystem.inline.hpp Normal file
View File

@ -0,0 +1,185 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_JFR_UTILITIES_JFRVERSIONSYSTEM_INLINE_HPP
#define SHARE_JFR_UTILITIES_JFRVERSIONSYSTEM_INLINE_HPP
#include "jfr/utilities/jfrVersionSystem.hpp"
#include "runtime/atomic.hpp"
#include "runtime/os.inline.hpp"
inline JfrVersionSystem::Node::Node() : _next(NULL), _version(0), _live(true) {}
inline traceid JfrVersionSystem::Node::version() const {
return _version;
}
inline void JfrVersionSystem::Node::set(traceid version) {
Atomic::release_store_fence(&_version, version);
}
inline JfrVersionSystem::JfrVersionSystem() : _tip(), _head(NULL) {
_tip._value = 1;
}
inline JfrVersionSystem::~JfrVersionSystem() {
reset();
}
inline void JfrVersionSystem::reset() {
NodePtr node = _head;
while (node != NULL) {
NodePtr next = node->_next;
delete node;
node = next;
}
_head = NULL;
_tip._value = 1;
}
inline JfrVersionSystem::Type JfrVersionSystem::tip() const {
return Atomic::load(&_tip._value);
}
inline JfrVersionSystem::Type JfrVersionSystem::increment() {
return Atomic::add(&_tip._value, (traceid)1);
}
inline JfrVersionSystem::NodePtr JfrVersionSystem::acquire() {
NodePtr node = _head;
// free
while (node != NULL) {
if (node->_live || Atomic::cmpxchg(&node->_live, false, true)) {
node = node->_next;
continue;
}
assert(node->_version == 0, "invariant");
return node;
}
// new
node = new Node();
NodePtr next;
do {
next = _head;
node->_next = next;
} while (Atomic::cmpxchg(&_head, next, node) != next);
return node;
}
inline void JfrVersionSystem::release(JfrVersionSystem::NodePtr node) {
assert(node != NULL, "invariant");
assert(node->_live, "invariant");
Atomic::release_store_fence(&node->_version, (traceid)0);
node->_live = false;
}
inline JfrVersionSystem::NodePtr
JfrVersionSystem::synchronize_with(JfrVersionSystem::Type version, JfrVersionSystem::NodePtr node) const {
assert(version <= tip(), "invariant");
while (node != NULL) {
const Type checkedout = Atomic::load_acquire(&node->_version);
if (checkedout > 0 && checkedout < version) {
return node;
}
node = node->_next;
}
return NULL;
}
inline void JfrVersionSystem::await(JfrVersionSystem::Type version) {
assert(version > 0, "invariant");
static const int backoff_unit_ns = 10;
int backoff_factor = 1;
NodePtr last = _head;
while (true) {
last = synchronize_with(version, last);
if (last == NULL) {
return;
}
os::naked_short_nanosleep(backoff_unit_ns * backoff_factor++);
}
}
inline JfrVersionSystem::Handle JfrVersionSystem::get_handle() {
return Handle(this);
}
inline JfrVersionSystem::Handle JfrVersionSystem::checkout_handle() {
Handle handle(this);
handle.checkout();
return handle;
}
inline JfrVersionSystem::Handle::Handle(JfrVersionSystem* system) : _system(system), _node(system->acquire()) {}
inline JfrVersionSystem::Handle::Handle() : _system(NULL), _node(NULL) {}
inline JfrVersionSystem::Handle::~Handle() {
if (_node != NULL) {
_system->release(_node);
}
}
inline void JfrVersionSystem::Handle::checkout() {
assert(_node != NULL, "invariant");
_node->set(_system->tip());
}
inline JfrVersionSystem::Type JfrVersionSystem::Handle::increment() {
assert(_node != NULL, "invariant");
const Type version = _system->increment();
assert(version > _node->version(), "invariant");
return version;
}
inline void JfrVersionSystem::Handle::release() {
assert(_node != NULL, "invariant");
_system->release(_node);
_node = NULL;
}
inline void JfrVersionSystem::Handle::await(JfrVersionSystem::Type version) {
_system->await(version);
}
#ifdef ASSERT
inline bool JfrVersionSystem::is_registered(JfrVersionSystem::Type version) const {
NodePtr node = _head;
while (node != NULL) {
if (Atomic::load_acquire(&node->_version) == version) {
return true;
}
node = node->_next;
}
return false;
}
inline bool JfrVersionSystem::Handle::is_tracked() const {
assert(_node != NULL, "invariant");
const Type current_version = _node->version();
return current_version != 0 && _system->is_registered(current_version);
}
#endif // ASSERT
#endif // SHARE_JFR_UTILITIES_JFRVERSIONSYSTEM_INLINE_HPP