From c66bef028963393077b7df7afb3ec2c8fdbe54aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20Gr=C3=B6nlund?= Date: Fri, 5 Jun 2020 15:11:44 +0200 Subject: [PATCH] 8233705: Let artifact iteration running time be a function of incrementally tagged artifacts Reviewed-by: egahlin --- .../jfrEventClassTransformer.cpp | 5 +- .../jfr/instrumentation/jfrJvmtiAgent.cpp | 6 +- src/hotspot/share/jfr/jfr.cpp | 2 +- src/hotspot/share/jfr/jni/jfrJniMethod.cpp | 11 +- src/hotspot/share/jfr/jni/jfrUpcalls.cpp | 4 +- .../checkpoint/objectSampleCheckpoint.cpp | 75 +-- .../checkpoint/objectSampleCheckpoint.hpp | 1 - .../checkpoint/jfrCheckpointManager.cpp | 213 +++---- .../checkpoint/jfrCheckpointManager.hpp | 51 +- .../checkpoint/jfrCheckpointWriter.cpp | 8 +- .../checkpoint/jfrCheckpointWriter.hpp | 7 +- .../recorder/checkpoint/jfrMetadataEvent.cpp | 7 +- .../recorder/checkpoint/jfrMetadataEvent.hpp | 4 +- .../recorder/checkpoint/types/jfrTypeSet.cpp | 126 ++-- .../recorder/checkpoint/types/jfrTypeSet.hpp | 3 +- .../checkpoint/types/jfrTypeSetUtils.cpp | 17 +- .../checkpoint/types/jfrTypeSetUtils.hpp | 22 +- .../checkpoint/types/traceid/jfrTraceId.cpp | 25 +- .../checkpoint/types/traceid/jfrTraceId.hpp | 34 +- .../types/traceid/jfrTraceId.inline.hpp | 125 ++-- .../types/traceid/jfrTraceIdBits.hpp | 62 ++ .../types/traceid/jfrTraceIdBits.inline.hpp | 172 ++++-- .../types/traceid/jfrTraceIdEpoch.hpp | 52 +- .../types/traceid/jfrTraceIdKlassQueue.cpp | 248 ++++++++ .../types/traceid/jfrTraceIdKlassQueue.hpp | 78 +++ .../types/traceid/jfrTraceIdLoadBarrier.cpp | 80 +++ .../types/traceid/jfrTraceIdLoadBarrier.hpp | 89 +++ .../traceid/jfrTraceIdLoadBarrier.inline.hpp | 127 ++++ .../types/traceid/jfrTraceIdMacros.hpp | 106 ++-- .../recorder/repository/jfrChunkWriter.cpp | 4 +- .../recorder/service/jfrRecorderService.cpp | 1 - .../recorder/service/jfrRecorderThread.cpp | 1 - .../jfr/recorder/stacktrace/jfrStackTrace.cpp | 4 +- .../jfr/recorder/storage/jfrEpochStorage.hpp | 83 +++ .../storage/jfrEpochStorage.inline.hpp | 112 ++++ .../storage/jfrFullStorage.inline.hpp | 3 + .../jfr/recorder/storage/jfrMemorySpace.hpp | 112 ++-- .../storage/jfrMemorySpace.inline.hpp | 548 +++++++++++------- .../storage/jfrMemorySpaceRetrieval.hpp | 32 +- .../share/jfr/recorder/storage/jfrStorage.cpp | 93 +-- .../share/jfr/recorder/storage/jfrStorage.hpp | 2 +- .../jfr/recorder/storage/jfrStorageUtils.hpp | 16 +- .../storage/jfrStorageUtils.inline.hpp | 26 + .../jfr/recorder/stringpool/jfrStringPool.cpp | 46 +- .../jfr/recorder/stringpool/jfrStringPool.hpp | 2 +- .../stringpool/jfrStringPoolBuffer.hpp | 5 - .../share/jfr/support/jfrEventClass.cpp | 87 --- .../jfrJdkJfrEvent.cpp} | 99 +++- .../{jfrEventClass.hpp => jfrJdkJfrEvent.hpp} | 21 +- .../share/jfr/support/jfrKlassUnloading.cpp | 131 +++++ .../jfrKlassUnloading.hpp} | 27 +- .../share/jfr/support/jfrMethodLookup.cpp | 1 + .../share/jfr/support/jfrThreadLocal.cpp | 18 +- .../share/jfr/support/jfrThreadLocal.hpp | 7 +- .../share/jfr/utilities/jfrEpochQueue.hpp | 76 +++ .../jfr/utilities/jfrEpochQueue.inline.hpp | 100 ++++ .../share/jfr/utilities/jfrPredicate.hpp | 62 ++ .../share/jfr/utilities/jfrRelation.hpp | 19 + src/hotspot/share/jfr/utilities/jfrTypes.hpp | 10 +- .../jfr/writers/jfrWriterHost.inline.hpp | 4 +- 60 files changed, 2484 insertions(+), 1028 deletions(-) create mode 100644 src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdBits.hpp create mode 100644 src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp create mode 100644 src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.hpp create mode 100644 src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.cpp create mode 100644 src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.hpp create mode 100644 src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.inline.hpp create mode 100644 src/hotspot/share/jfr/recorder/storage/jfrEpochStorage.hpp create mode 100644 src/hotspot/share/jfr/recorder/storage/jfrEpochStorage.inline.hpp delete mode 100644 src/hotspot/share/jfr/support/jfrEventClass.cpp rename src/hotspot/share/jfr/{jni/jfrGetAllEventClasses.cpp => support/jfrJdkJfrEvent.cpp} (74%) rename src/hotspot/share/jfr/support/{jfrEventClass.hpp => jfrJdkJfrEvent.hpp} (75%) create mode 100644 src/hotspot/share/jfr/support/jfrKlassUnloading.cpp rename src/hotspot/share/jfr/{jni/jfrGetAllEventClasses.hpp => support/jfrKlassUnloading.hpp} (65%) create mode 100644 src/hotspot/share/jfr/utilities/jfrEpochQueue.hpp create mode 100644 src/hotspot/share/jfr/utilities/jfrEpochQueue.inline.hpp create mode 100644 src/hotspot/share/jfr/utilities/jfrPredicate.hpp diff --git a/src/hotspot/share/jfr/instrumentation/jfrEventClassTransformer.cpp b/src/hotspot/share/jfr/instrumentation/jfrEventClassTransformer.cpp index 60437a0bf1a..793ee757879 100644 --- a/src/hotspot/share/jfr/instrumentation/jfrEventClassTransformer.cpp +++ b/src/hotspot/share/jfr/instrumentation/jfrEventClassTransformer.cpp @@ -37,7 +37,8 @@ #include "jfr/jfr.hpp" #include "jfr/jni/jfrJavaSupport.hpp" #include "jfr/jni/jfrUpcalls.hpp" -#include "jfr/support/jfrEventClass.hpp" +#include "jfr/recorder/checkpoint/types/traceid/jfrTraceId.inline.hpp" +#include "jfr/support/jfrJdkJfrEvent.hpp" #include "jfr/utilities/jfrBigEndian.hpp" #include "jfr/writers/jfrBigEndianWriter.hpp" #include "logging/log.hpp" @@ -1408,7 +1409,7 @@ static ClassFileStream* create_new_bytes_for_subklass(const InstanceKlass* ik, c jint size_instrumented_data = 0; unsigned char* instrumented_data = NULL; const jclass super = (jclass)JNIHandles::make_local(ik->super()->java_mirror()); - JfrUpcalls::new_bytes_eager_instrumentation(TRACE_ID(ik), + JfrUpcalls::new_bytes_eager_instrumentation(JfrTraceId::load_raw(ik), force_instrumentation, super, size_of_new_bytes, diff --git a/src/hotspot/share/jfr/instrumentation/jfrJvmtiAgent.cpp b/src/hotspot/share/jfr/instrumentation/jfrJvmtiAgent.cpp index ef481d50775..100d9deb371 100644 --- a/src/hotspot/share/jfr/instrumentation/jfrJvmtiAgent.cpp +++ b/src/hotspot/share/jfr/instrumentation/jfrJvmtiAgent.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,7 +29,7 @@ #include "jfr/jni/jfrUpcalls.hpp" #include "jfr/recorder/checkpoint/types/traceid/jfrTraceId.inline.hpp" #include "jfr/recorder/service/jfrOptionSet.hpp" -#include "jfr/support/jfrEventClass.hpp" +#include "jfr/support/jfrJdkJfrEvent.hpp" #include "logging/log.hpp" #include "memory/resourceArea.hpp" #include "prims/jvmtiEnvBase.hpp" @@ -91,7 +91,7 @@ extern "C" void JNICALL jfr_on_class_file_load_hook(jvmtiEnv *jvmti_env, JavaThread* jt = JavaThread::thread_from_jni_environment(jni_env); DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(jt));; ThreadInVMfromNative tvmfn(jt); - JfrUpcalls::on_retransform(JfrTraceId::get(class_being_redefined), + JfrUpcalls::on_retransform(JfrTraceId::load_raw(class_being_redefined), class_being_redefined, class_data_len, class_data, diff --git a/src/hotspot/share/jfr/jfr.cpp b/src/hotspot/share/jfr/jfr.cpp index 77eb2263a48..406f811491f 100644 --- a/src/hotspot/share/jfr/jfr.cpp +++ b/src/hotspot/share/jfr/jfr.cpp @@ -66,7 +66,7 @@ void Jfr::on_create_vm_3() { void Jfr::on_unloading_classes() { if (JfrRecorder::is_created()) { - JfrCheckpointManager::write_type_set_for_unloaded_classes(); + JfrCheckpointManager::on_unloading_classes(); } } diff --git a/src/hotspot/share/jfr/jni/jfrJniMethod.cpp b/src/hotspot/share/jfr/jni/jfrJniMethod.cpp index 39f3b5eb8b7..5e481740399 100644 --- a/src/hotspot/share/jfr/jni/jfrJniMethod.cpp +++ b/src/hotspot/share/jfr/jni/jfrJniMethod.cpp @@ -38,12 +38,13 @@ #include "jfr/recorder/service/jfrOptionSet.hpp" #include "jfr/recorder/stacktrace/jfrStackTraceRepository.hpp" #include "jfr/recorder/stringpool/jfrStringPool.hpp" -#include "jfr/jni/jfrGetAllEventClasses.hpp" #include "jfr/jni/jfrJavaSupport.hpp" #include "jfr/jni/jfrJniMethodRegistration.hpp" #include "jfr/instrumentation/jfrEventClassTransformer.hpp" #include "jfr/instrumentation/jfrJvmtiAgent.hpp" #include "jfr/leakprofiler/leakProfiler.hpp" +#include "jfr/support/jfrJdkJfrEvent.hpp" +#include "jfr/support/jfrKlassUnloading.hpp" #include "jfr/utilities/jfrJavaLog.hpp" #include "jfr/utilities/jfrTimeConverter.hpp" #include "jfr/utilities/jfrTime.hpp" @@ -163,7 +164,7 @@ NO_TRANSITION(jlong, jfr_get_epoch_address(JNIEnv* env, jobject jvm)) NO_TRANSITION_END NO_TRANSITION(jlong, jfr_get_unloaded_event_classes_count(JNIEnv* env, jobject jvm)) - return JfrEventClasses::unloaded_event_classes_count(); + return JfrKlassUnloading::event_class_count(); NO_TRANSITION_END NO_TRANSITION(jdouble, jfr_time_conv_factor(JNIEnv* env, jobject jvm)) @@ -234,11 +235,11 @@ JVM_ENTRY_NO_ENV(jboolean, jfr_emit_event(JNIEnv* env, jobject jvm, jlong eventT JVM_END JVM_ENTRY_NO_ENV(jobject, jfr_get_all_event_classes(JNIEnv* env, jobject jvm)) - return JfrEventClasses::get_all_event_classes(thread); + return JdkJfrEvent::get_all_klasses(thread); JVM_END JVM_ENTRY_NO_ENV(jlong, jfr_class_id(JNIEnv* env, jclass jvm, jclass jc)) - return JfrTraceId::use(jc); + return JfrTraceId::load(jc); JVM_END JVM_ENTRY_NO_ENV(jlong, jfr_stacktrace_id(JNIEnv* env, jobject jvm, jint skip)) @@ -311,7 +312,7 @@ JVM_ENTRY_NO_ENV(void, jfr_abort(JNIEnv* env, jobject jvm, jstring errorMsg)) JVM_END JVM_ENTRY_NO_ENV(jlong, jfr_type_id(JNIEnv* env, jobject jvm, jclass jc)) - return JfrTraceId::get(jc); + return JfrTraceId::load_raw(jc); JVM_END JVM_ENTRY_NO_ENV(jboolean, jfr_add_string_constant(JNIEnv* env, jclass jvm, jboolean epoch, jlong id, jstring string)) diff --git a/src/hotspot/share/jfr/jni/jfrUpcalls.cpp b/src/hotspot/share/jfr/jni/jfrUpcalls.cpp index 7b2177c903b..43c52c6ef81 100644 --- a/src/hotspot/share/jfr/jni/jfrUpcalls.cpp +++ b/src/hotspot/share/jfr/jni/jfrUpcalls.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,7 +28,7 @@ #include "classfile/systemDictionary.hpp" #include "jfr/jni/jfrJavaSupport.hpp" #include "jfr/jni/jfrUpcalls.hpp" -#include "jfr/support/jfrEventClass.hpp" +#include "jfr/support/jfrJdkJfrEvent.hpp" #include "logging/log.hpp" #include "memory/oopFactory.hpp" #include "oops/oop.inline.hpp" diff --git a/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.cpp b/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.cpp index bc1e6108e8d..1ac1ebf54e4 100644 --- a/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.cpp +++ b/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.cpp @@ -35,36 +35,15 @@ #include "jfr/recorder/checkpoint/types/traceid/jfrTraceId.inline.hpp" #include "jfr/recorder/service/jfrOptionSet.hpp" #include "jfr/recorder/stacktrace/jfrStackTraceRepository.hpp" +#include "jfr/support/jfrKlassUnloading.hpp" #include "jfr/support/jfrMethodLookup.hpp" #include "jfr/utilities/jfrHashtable.hpp" -#include "jfr/utilities/jfrTypes.hpp" +#include "jfr/utilities/jfrPredicate.hpp" +#include "jfr/utilities/jfrRelation.hpp" #include "oops/instanceKlass.inline.hpp" #include "runtime/mutexLocker.hpp" #include "runtime/safepoint.hpp" #include "runtime/thread.hpp" -#include "utilities/growableArray.hpp" - -static bool predicate(GrowableArray* set, traceid id) { - assert(set != NULL, "invariant"); - bool found = false; - set->find_sorted(id, found); - return found; -} - -static bool mutable_predicate(GrowableArray* set, traceid id) { - assert(set != NULL, "invariant"); - bool found = false; - const int location = set->find_sorted(id, found); - if (!found) { - set->insert_before(location, id); - } - return found; -} - -static bool add(GrowableArray* set, traceid id) { - assert(set != NULL, "invariant"); - return mutable_predicate(set, id); -} const int initial_array_size = 64; @@ -87,7 +66,12 @@ Semaphore ThreadIdExclusiveAccess::_mutex_semaphore(1); static bool has_thread_exited(traceid tid) { assert(tid != 0, "invariant"); - return unloaded_thread_id_set != NULL && predicate(unloaded_thread_id_set, tid); + return unloaded_thread_id_set != NULL && JfrPredicate::test(unloaded_thread_id_set, tid); +} + +static bool add(GrowableArray* set, traceid id) { + assert(set != NULL, "invariant"); + return JfrMutablePredicate::test(set, id); } static void add_to_unloaded_thread_set(traceid tid) { @@ -105,31 +89,6 @@ void ObjectSampleCheckpoint::on_thread_exit(JavaThread* jt) { } } -// Track the set of unloaded klasses during a chunk / epoch. -// Methods in stacktraces belonging to unloaded klasses must not be accessed. -static GrowableArray* unloaded_klass_set = NULL; - -static void add_to_unloaded_klass_set(traceid klass_id) { - assert_locked_or_safepoint(ClassLoaderDataGraph_lock); - if (unloaded_klass_set == NULL) { - unloaded_klass_set = c_heap_allocate_array(); - } - unloaded_klass_set->append(klass_id); -} - -static void sort_unloaded_klass_set() { - assert_locked_or_safepoint(ClassLoaderDataGraph_lock); - if (unloaded_klass_set != NULL && unloaded_klass_set->length() > 1) { - unloaded_klass_set->sort(sort_traceid); - } -} - -void ObjectSampleCheckpoint::on_klass_unload(const Klass* k) { - assert_locked_or_safepoint(ClassLoaderDataGraph_lock); - assert(k != NULL, "invariant"); - add_to_unloaded_klass_set(JfrTraceId::get(k)); -} - template static void do_samples(ObjectSample* sample, const ObjectSample* end, Processor& processor) { assert(sample != NULL, "invariant"); @@ -228,7 +187,6 @@ static GrowableArray* id_set = NULL; static void prepare_for_resolution() { id_set = new GrowableArray(JfrOptionSet::old_object_queue_size()); - sort_unloaded_klass_set(); } static bool stack_trace_precondition(const ObjectSample* sample) { @@ -290,6 +248,7 @@ static void install_stack_traces(const ObjectSampler* sampler, JfrStackTraceRepo assert(sampler != NULL, "invariant"); const ObjectSample* const last = sampler->last(); if (last != sampler->last_resolved()) { + JfrKlassUnloading::sort(); StackTraceBlobInstaller installer(stack_trace_repo); iterate_samples(installer); } @@ -307,13 +266,13 @@ void ObjectSampleCheckpoint::on_rotation(const ObjectSampler* sampler, JfrStackT static bool is_klass_unloaded(traceid klass_id) { assert(ClassLoaderDataGraph_lock->owned_by_self(), "invariant"); - return unloaded_klass_set != NULL && predicate(unloaded_klass_set, klass_id); + return JfrKlassUnloading::is_unloaded(klass_id); } static bool is_processed(traceid method_id) { assert(method_id != 0, "invariant"); assert(id_set != NULL, "invariant"); - return mutable_predicate(id_set, method_id); + return JfrMutablePredicate::test(id_set, method_id); } void ObjectSampleCheckpoint::add_to_leakp_set(const InstanceKlass* ik, traceid method_id) { @@ -324,7 +283,7 @@ void ObjectSampleCheckpoint::add_to_leakp_set(const InstanceKlass* ik, traceid m const Method* const method = JfrMethodLookup::lookup(ik, method_id); assert(method != NULL, "invariant"); assert(method->method_holder() == ik, "invariant"); - JfrTraceId::set_leakp(ik, method); + JfrTraceId::load_leakp(ik, method); } void ObjectSampleCheckpoint::write_stacktrace(const JfrStackTrace* trace, JfrCheckpointWriter& writer) { @@ -419,13 +378,6 @@ void ObjectSampleCheckpoint::write(const ObjectSampler* sampler, EdgeStore* edge } } -static void clear_unloaded_klass_set() { - assert(ClassLoaderDataGraph_lock->owned_by_self(), "invariant"); - if (unloaded_klass_set != NULL && unloaded_klass_set->is_nonempty()) { - unloaded_klass_set->clear(); - } -} - // A linked list of saved type set blobs for the epoch. // The link consist of a reference counted handle. static JfrBlobHandle saved_type_set_blobs; @@ -433,7 +385,6 @@ static JfrBlobHandle saved_type_set_blobs; static void release_state_for_previous_epoch() { // decrements the reference count and the list is reinitialized saved_type_set_blobs = JfrBlobHandle(); - clear_unloaded_klass_set(); } class BlobInstaller { diff --git a/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.hpp b/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.hpp index c8cd972400e..ff08d9a5336 100644 --- a/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.hpp +++ b/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.hpp @@ -50,7 +50,6 @@ class ObjectSampleCheckpoint : AllStatic { static void write_stacktrace(const JfrStackTrace* trace, JfrCheckpointWriter& writer); static void write(const ObjectSampler* sampler, EdgeStore* edge_store, bool emit_all, Thread* thread); public: - static void on_klass_unload(const Klass* k); static void on_type_set(JfrCheckpointWriter& writer); static void on_type_set_unload(JfrCheckpointWriter& writer); static void on_thread_exit(JavaThread* jt); diff --git a/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp b/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp index c93e888bfff..55e94860258 100644 --- a/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp +++ b/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp @@ -36,6 +36,7 @@ #include "jfr/recorder/service/jfrOptionSet.hpp" #include "jfr/recorder/storage/jfrMemorySpace.inline.hpp" #include "jfr/recorder/storage/jfrStorageUtils.inline.hpp" +#include "jfr/support/jfrKlassUnloading.hpp" #include "jfr/utilities/jfrBigEndian.hpp" #include "jfr/utilities/jfrIterator.hpp" #include "jfr/utilities/jfrLinkedList.inline.hpp" @@ -88,46 +89,43 @@ void JfrCheckpointManager::destroy() { } JfrCheckpointManager::JfrCheckpointManager(JfrChunkWriter& cw) : - _free_list_mspace(NULL), - _epoch_transition_mspace(NULL), - _service_thread(NULL), - _chunkwriter(cw), - _checkpoint_epoch_state(JfrTraceIdEpoch::epoch()) {} + _mspace(NULL), + _chunkwriter(cw) {} JfrCheckpointManager::~JfrCheckpointManager() { - if (_free_list_mspace != NULL) { - delete _free_list_mspace; - } - if (_epoch_transition_mspace != NULL) { - delete _epoch_transition_mspace; - } + JfrTraceIdLoadBarrier::destroy(); JfrTypeManager::destroy(); + delete _mspace; } -static const size_t unlimited_mspace_size = 0; -static const size_t checkpoint_buffer_cache_count = 2; -static const size_t checkpoint_buffer_size = 512 * K; +static const size_t buffer_count = 2; +static const size_t buffer_size = 512 * K; -static JfrCheckpointMspace* allocate_mspace(size_t size, size_t limit, size_t cache_count, JfrCheckpointManager* mgr) { - return create_mspace(size, limit, cache_count, mgr); +static JfrCheckpointMspace* allocate_mspace(size_t min_elem_size, + size_t free_list_cache_count_limit, + size_t cache_prealloc_count, + bool prealloc_to_free_list, + JfrCheckpointManager* mgr) { + return create_mspace(min_elem_size, + free_list_cache_count_limit, + cache_prealloc_count, + prealloc_to_free_list, + mgr); } bool JfrCheckpointManager::initialize() { - assert(_free_list_mspace == NULL, "invariant"); - _free_list_mspace = allocate_mspace(checkpoint_buffer_size, unlimited_mspace_size, checkpoint_buffer_cache_count, this); - if (_free_list_mspace == NULL) { + assert(_mspace == NULL, "invariant"); + _mspace = allocate_mspace(buffer_size, 0, 0, false, this); // post-pone preallocation + if (_mspace == NULL) { return false; } - assert(_epoch_transition_mspace == NULL, "invariant"); - _epoch_transition_mspace = allocate_mspace(checkpoint_buffer_size, unlimited_mspace_size, checkpoint_buffer_cache_count, this); - if (_epoch_transition_mspace == NULL) { - return false; + // preallocate buffer count to each of the epoch live lists + for (size_t i = 0; i < buffer_count * 2; ++i) { + Buffer* const buffer = mspace_allocate(buffer_size, _mspace); + _mspace->add_to_live_list(buffer, i % 2 == 0); } - return JfrTypeManager::initialize(); -} - -void JfrCheckpointManager::register_service_thread(const Thread* thread) { - _service_thread = thread; + assert(_mspace->free_list_is_empty(), "invariant"); + return JfrTypeManager::initialize() && JfrTraceIdLoadBarrier::initialize(); } void JfrCheckpointManager::register_full(BufferPtr buffer, Thread* thread) { @@ -151,46 +149,36 @@ static void assert_release(const BufferPtr buffer) { } #endif // ASSERT -bool JfrCheckpointManager::use_epoch_transition_mspace(const Thread* thread) const { - return _service_thread != thread && Atomic::load_acquire(&_checkpoint_epoch_state) != JfrTraceIdEpoch::epoch(); -} - -static const size_t lease_retry = 10; - -BufferPtr JfrCheckpointManager::lease(JfrCheckpointMspace* mspace, Thread* thread, size_t size /* 0 */) { +static BufferPtr lease(size_t size, JfrCheckpointMspace* mspace, size_t retry_count, Thread* thread, bool previous_epoch) { assert(mspace != NULL, "invariant"); - static const size_t max_elem_size = mspace->min_elem_size(); // min is max + static const size_t max_elem_size = mspace->min_element_size(); // min is max BufferPtr buffer; if (size <= max_elem_size) { - buffer = mspace_get_free_lease_with_retry(size, mspace, lease_retry, thread); + buffer = mspace_acquire_lease_with_retry(size, mspace, retry_count, thread, previous_epoch); if (buffer != NULL) { DEBUG_ONLY(assert_lease(buffer);) return buffer; } } - buffer = mspace_allocate_transient_lease_to_full(size, mspace, thread); + buffer = mspace_allocate_transient_lease_to_live_list(size, mspace, thread, previous_epoch); DEBUG_ONLY(assert_lease(buffer);) return buffer; } -BufferPtr JfrCheckpointManager::lease(Thread* thread, size_t size /* 0 */) { - JfrCheckpointManager& manager = instance(); - JfrCheckpointMspace* const mspace = manager.use_epoch_transition_mspace(thread) ? - manager._epoch_transition_mspace : - manager._free_list_mspace; - return lease(mspace, thread, size); +static const size_t lease_retry = 100; + +BufferPtr JfrCheckpointManager::lease(Thread* thread, bool previous_epoch /* false */, size_t size /* 0 */) { + return ::lease(size, instance()._mspace, lease_retry, thread, previous_epoch); } -JfrCheckpointMspace* JfrCheckpointManager::lookup(BufferPtr old) const { +bool JfrCheckpointManager::lookup(BufferPtr old) const { assert(old != NULL, "invariant"); - return _free_list_mspace->in_mspace(old) ? _free_list_mspace : _epoch_transition_mspace; + return !_mspace->in_current_epoch_list(old); } BufferPtr JfrCheckpointManager::lease(BufferPtr old, Thread* thread, size_t size /* 0 */) { assert(old != NULL, "invariant"); - JfrCheckpointMspace* mspace = instance().lookup(old); - assert(mspace != NULL, "invariant"); - return lease(mspace, thread, size); + return ::lease(size, instance()._mspace, lease_retry, thread, instance().lookup(old)); } /* @@ -320,23 +308,9 @@ class CheckpointWriteOp { }; typedef CheckpointWriteOp WriteOperation; -typedef ReleaseOp CheckpointReleaseFreeOperation; -typedef ScavengingReleaseOp CheckpointReleaseFullOperation; - -template