diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp index 9e67b27ad38..1f76fa0778a 100644 --- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp @@ -219,6 +219,10 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register t1, bind(inflated); const Register t1_monitor = t1; + // Offsets into the current thread's object monitor cache (omc). + const ByteSize thr_omc_offset = JavaThread::om_cache_offset(); + const ByteSize omc_monitor_offset = OMCache::monitor_offset(); + const ByteSize omc_obj_offset = OMCache::obj_offset(); if (!UseObjectMonitorTable) { assert(t1_monitor == t1_mark, "should be the same here"); @@ -229,18 +233,12 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register t1, // Save the mark, we might need it to extract the hash. mov(t3, t1_mark); - // Look for the monitor in the om_cache. + // Look for the monitor in the current thread's object monitor cache (omc). - ByteSize cache_offset = JavaThread::om_cache_oops_offset(); - ByteSize monitor_offset = OMCache::oop_to_monitor_difference(); - const int num_unrolled = OMCache::CAPACITY; - for (int i = 0; i < num_unrolled; i++) { - ldr(t1_monitor, Address(rthread, cache_offset + monitor_offset)); - ldr(t2, Address(rthread, cache_offset)); - cmp(obj, t2); - br(Assembler::EQ, monitor_found); - cache_offset = cache_offset + OMCache::oop_to_oop_difference(); - } + ldr(t1_monitor, Address(rthread, thr_omc_offset + omc_monitor_offset)); + ldr(t2, Address(rthread, thr_omc_offset + omc_obj_offset)); + cmp(obj, t2); + br(Assembler::EQ, monitor_found); // Look for the monitor in the table. @@ -268,6 +266,10 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register t1, cmp(t3, obj); br(Assembler::NE, slow_path); + // Store the monitor in the current thread's object monitor cache (omc). + str(t1_monitor, Address(rthread, thr_omc_offset + omc_monitor_offset)); + str(obj, Address(rthread, thr_omc_offset + omc_obj_offset)); + bind(monitor_found); } @@ -296,6 +298,7 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register t1, bind(monitor_locked); if (UseObjectMonitorTable) { + // Cache the monitor for unlock. str(t1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes())); } } diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp index 1501934d48f..54327b738d6 100644 --- a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp @@ -2763,6 +2763,11 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register const Register monitor = UseObjectMonitorTable ? tmp1 : noreg; const Register owner_addr = tmp2; const Register thread_id = UseObjectMonitorTable ? tmp3 : tmp1; + // Offsets into the current thread's object monitor cache (omc). + const ByteSize thr_omc_offset = JavaThread::om_cache_offset(); + const ByteSize omc_monitor_offset = OMCache::monitor_offset(); + const ByteSize omc_obj_offset = OMCache::obj_offset(); + Label monitor_locked; if (!UseObjectMonitorTable) { @@ -2777,18 +2782,12 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register // Save the mark, we might need it to extract the hash. mr(tmp2_hash, mark); - // Look for the monitor in the om_cache. + // Look for the monitor in the current thread's object monitor cache (omc). - ByteSize cache_offset = JavaThread::om_cache_oops_offset(); - ByteSize monitor_offset = OMCache::oop_to_monitor_difference(); - const int num_unrolled = OMCache::CAPACITY; - for (int i = 0; i < num_unrolled; i++) { - ld(R0, in_bytes(cache_offset), R16_thread); - ld(monitor, in_bytes(cache_offset + monitor_offset), R16_thread); - cmpd(CR0, R0, obj); - beq(CR0, monitor_found); - cache_offset = cache_offset + OMCache::oop_to_oop_difference(); - } + ld(R0, in_bytes(thr_omc_offset + omc_obj_offset), R16_thread); + ld(monitor, in_bytes(thr_omc_offset + omc_monitor_offset), R16_thread); + cmpd(CR0, R0, obj); + beq(CR0, monitor_found); // Look for the monitor in the table. @@ -2817,6 +2816,10 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register cmpd(CR0, tmp3, obj); bne(CR0, slow_path); + // Store the monitor in the current thread's object monitor cache (omc). + std(monitor, in_bytes(thr_omc_offset + omc_monitor_offset), R16_thread); + std(obj, in_bytes(thr_omc_offset + omc_obj_offset), R16_thread); + bind(monitor_found); // Compute owner address. @@ -2854,6 +2857,7 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register bind(monitor_locked); if (UseObjectMonitorTable) { + // Cache the monitor for unlock. std(monitor, BasicLock::object_monitor_cache_offset_in_bytes(), box); } } diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp index b658e36bd59..2dca5215aec 100644 --- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp @@ -121,6 +121,10 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, bind(inflated); const Register tmp1_monitor = tmp1; + // Offsets into the current thread's object monitor cache (omc). + const ByteSize thr_omc_offset = JavaThread::om_cache_offset(); + const ByteSize omc_monitor_offset = OMCache::monitor_offset(); + const ByteSize omc_obj_offset = OMCache::obj_offset(); if (!UseObjectMonitorTable) { assert(tmp1_monitor == tmp1_mark, "should be the same here"); @@ -132,17 +136,11 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, // Save the mark, we might need it to extract the hash. mv(tmp2_hash, tmp1_mark); - // Look for the monitor in the om_cache. + // Look for the monitor in the current thread's object monitor cache (omc). - ByteSize cache_offset = JavaThread::om_cache_oops_offset(); - ByteSize monitor_offset = OMCache::oop_to_monitor_difference(); - const int num_unrolled = OMCache::CAPACITY; - for (int i = 0; i < num_unrolled; i++) { - ld(tmp1_monitor, Address(xthread, cache_offset + monitor_offset)); - ld(tmp4, Address(xthread, cache_offset)); - beq(obj, tmp4, monitor_found); - cache_offset = cache_offset + OMCache::oop_to_oop_difference(); - } + ld(tmp1_monitor, Address(xthread, thr_omc_offset + omc_monitor_offset)); + ld(tmp4, Address(xthread, thr_omc_offset + omc_obj_offset)); + beq(obj, tmp4, monitor_found); // Look for the monitor in the table. @@ -170,6 +168,10 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, bs_asm->try_peek_weak_handle_in_nmethod(this, tmp3, tmp3, tmp2, slow_path); bne(tmp3, obj, slow_path); + // Store the monitor in the current thread's object monitor cache (omc). + sd(tmp1_monitor, Address(xthread, thr_omc_offset + omc_monitor_offset)); + sd(obj, Address(xthread, thr_omc_offset + omc_obj_offset)); + bind(monitor_found); } @@ -200,6 +202,7 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, bind(monitor_locked); if (UseObjectMonitorTable) { + // Cache the monitor for unlock. sd(tmp1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes())); } } diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.cpp b/src/hotspot/cpu/s390/macroAssembler_s390.cpp index 6eb14452401..a3a25e07197 100644 --- a/src/hotspot/cpu/s390/macroAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/macroAssembler_s390.cpp @@ -6382,6 +6382,11 @@ void MacroAssembler::compiler_fast_lock_object(Register obj, Register box, Regis bind(inflated); const Register tmp1_monitor = tmp1; + // Offsets into the current thread's object monitor cache (omc). + const ByteSize thr_omc_offset = JavaThread::om_cache_offset(); + const ByteSize omc_monitor_offset = OMCache::monitor_offset(); + const ByteSize omc_obj_offset = OMCache::obj_offset(); + if (!UseObjectMonitorTable) { assert(tmp1_monitor == mark, "should be the same here"); } else { @@ -6392,17 +6397,11 @@ void MacroAssembler::compiler_fast_lock_object(Register obj, Register box, Regis // Save the mark, we might need it to extract the hash. z_lgr(hash, mark); - // Look for the monitor in the om_cache. + // Look for the monitor in the current thread's object monitor cache (omc). - ByteSize cache_offset = JavaThread::om_cache_oops_offset(); - ByteSize monitor_offset = OMCache::oop_to_monitor_difference(); - const int num_unrolled = OMCache::CAPACITY; - for (int i = 0; i < num_unrolled; i++) { - z_lg(tmp1_monitor, Address(Z_thread, cache_offset + monitor_offset)); - z_cg(obj, Address(Z_thread, cache_offset)); - z_bre(monitor_found); - cache_offset = cache_offset + OMCache::oop_to_oop_difference(); - } + z_lg(tmp1_monitor, Address(Z_thread, thr_omc_offset + omc_monitor_offset)); + z_cg(obj, Address(Z_thread, thr_omc_offset + omc_obj_offset)); + z_bre(monitor_found); // Get the hash code. z_srlg(hash, hash, markWord::hash_shift); @@ -6429,6 +6428,10 @@ void MacroAssembler::compiler_fast_lock_object(Register obj, Register box, Regis z_cgr(obj, tmp2); z_brne(slow_path); + // Store the monitor in the current thread's object monitor cache (omc). + z_stg(tmp1_monitor, Address(Z_thread, thr_omc_offset + omc_monitor_offset)); + z_stg(obj, Address(Z_thread, thr_omc_offset + omc_obj_offset)); + bind(monitor_found); } NearLabel monitor_locked; @@ -6458,7 +6461,7 @@ void MacroAssembler::compiler_fast_lock_object(Register obj, Register box, Regis bind(monitor_locked); if (UseObjectMonitorTable) { - // Cache the monitor for unlock + // Cache the monitor for unlock. z_stg(tmp1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes())); } // set the CC now diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp index 69308bb2a7e..2e253ff142c 100644 --- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp @@ -294,6 +294,10 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register rax_reg, bind(inflated); const Register monitor = t; + // Offsets into the current thread's object monitor cache (omc). + const ByteSize thr_omc_offset = JavaThread::om_cache_offset(); + const ByteSize omc_monitor_offset = OMCache::monitor_offset(); + const ByteSize omc_obj_offset = OMCache::obj_offset(); if (!UseObjectMonitorTable) { assert(mark == monitor, "should be the same here"); @@ -301,17 +305,11 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register rax_reg, const Register hash = t; Label monitor_found; - // Look for the monitor in the om_cache. + // Look for the monitor in the current thread's object monitor cache (omc). - ByteSize cache_offset = JavaThread::om_cache_oops_offset(); - ByteSize monitor_offset = OMCache::oop_to_monitor_difference(); - const int num_unrolled = OMCache::CAPACITY; - for (int i = 0; i < num_unrolled; i++) { - movptr(monitor, Address(thread, cache_offset + monitor_offset)); - cmpptr(obj, Address(thread, cache_offset)); - jccb(Assembler::equal, monitor_found); - cache_offset = cache_offset + OMCache::oop_to_oop_difference(); - } + movptr(monitor, Address(thread, thr_omc_offset + omc_monitor_offset)); + cmpptr(obj, Address(thread, thr_omc_offset + omc_obj_offset)); + jccb(Assembler::equal, monitor_found); // Look for the monitor in the table. @@ -340,6 +338,10 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register rax_reg, cmpptr(rax_reg, obj); jcc(Assembler::notEqual, slow_path); + // Store the monitor in the current thread's object monitor cache (omc). + movptr(Address(thread, thr_omc_offset + omc_monitor_offset), monitor); + movptr(Address(thread, thr_omc_offset + omc_obj_offset), obj); + bind(monitor_found); } const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast(markWord::monitor_value)); diff --git a/src/hotspot/share/runtime/javaThread.hpp b/src/hotspot/share/runtime/javaThread.hpp index cd757ed8ec1..13e079f1be5 100644 --- a/src/hotspot/share/runtime/javaThread.hpp +++ b/src/hotspot/share/runtime/javaThread.hpp @@ -1238,7 +1238,6 @@ public: static ByteSize lock_stack_base_offset() { return lock_stack_offset() + LockStack::base_offset(); } static ByteSize om_cache_offset() { return byte_offset_of(JavaThread, _om_cache); } - static ByteSize om_cache_oops_offset() { return om_cache_offset() + OMCache::entries_offset(); } void om_set_monitor_cache(ObjectMonitor* monitor); void om_clear_monitor_cache(); diff --git a/src/hotspot/share/runtime/lockStack.cpp b/src/hotspot/share/runtime/lockStack.cpp index 58b9c58a329..a0f20b4fd31 100644 --- a/src/hotspot/share/runtime/lockStack.cpp +++ b/src/hotspot/share/runtime/lockStack.cpp @@ -115,7 +115,6 @@ void LockStack::print_on(outputStream* st) { } } -OMCache::OMCache(JavaThread* jt) : _entries() { +OMCache::OMCache(JavaThread* jt) { STATIC_ASSERT(std::is_standard_layout::value); - STATIC_ASSERT(std::is_standard_layout::value); } diff --git a/src/hotspot/share/runtime/lockStack.hpp b/src/hotspot/share/runtime/lockStack.hpp index 8d7e3644efa..cc76005cf4c 100644 --- a/src/hotspot/share/runtime/lockStack.hpp +++ b/src/hotspot/share/runtime/lockStack.hpp @@ -130,19 +130,14 @@ class LockStack { class OMCache { friend class VMStructs; - public: - static constexpr int CAPACITY = 2; private: - struct OMCacheEntry { - oop _oop = nullptr; - ObjectMonitor* _monitor = nullptr; - } _entries[CAPACITY]; + oop _obj = nullptr; + ObjectMonitor* _monitor = nullptr; public: - static ByteSize entries_offset() { return byte_offset_of(OMCache, _entries); } - static constexpr ByteSize oop_to_oop_difference() { return in_ByteSize(sizeof(OMCacheEntry)); } - static constexpr ByteSize oop_to_monitor_difference() { return in_ByteSize(sizeof(oop)); } + static constexpr ByteSize obj_offset() { return byte_offset_of(OMCache, _obj); } + static constexpr ByteSize monitor_offset() { return byte_offset_of(OMCache, _monitor); } explicit OMCache(JavaThread* jt); diff --git a/src/hotspot/share/runtime/lockStack.inline.hpp b/src/hotspot/share/runtime/lockStack.inline.hpp index a9ad3553db8..cb5c4de90b5 100644 --- a/src/hotspot/share/runtime/lockStack.inline.hpp +++ b/src/hotspot/share/runtime/lockStack.inline.hpp @@ -250,53 +250,30 @@ inline void LockStack::oops_do(OopClosure* cl) { } inline void OMCache::set_monitor(ObjectMonitor *monitor) { - const int end = OMCache::CAPACITY - 1; - oop obj = monitor->object_peek(); assert(obj != nullptr, "must be alive"); assert(monitor == ObjectSynchronizer::get_monitor_from_table(obj), "must exist in table"); - OMCacheEntry to_insert = {obj, monitor}; - - for (int i = 0; i < end; ++i) { - if (_entries[i]._oop == obj || - _entries[i]._monitor == nullptr || - _entries[i]._monitor->is_being_async_deflated()) { - // Use stale slot. - _entries[i] = to_insert; - return; - } - // Swap with the most recent value. - ::swap(to_insert, _entries[i]); - } - _entries[end] = to_insert; + _monitor = monitor; + _obj = obj; } inline ObjectMonitor* OMCache::get_monitor(oop o) { - for (int i = 0; i < CAPACITY; ++i) { - if (_entries[i]._oop == o) { - assert(_entries[i]._monitor != nullptr, "monitor must exist"); - if (_entries[i]._monitor->is_being_async_deflated()) { - // Bad monitor - // Shift down rest - for (; i < CAPACITY - 1; ++i) { - _entries[i] = _entries[i + 1]; - } - // Clear end - _entries[i] = {}; - return nullptr; - } - return _entries[i]._monitor; + if (_obj == o) { + assert(_monitor != nullptr, "monitor must exist"); + if (!_monitor->is_being_async_deflated()) { + return _monitor; } + // Bad monitor, so clear the cache. + _obj = nullptr; + _monitor = nullptr; } return nullptr; } inline void OMCache::clear() { - for (size_t i = 0; i < CAPACITY; ++i) { - // Clear - _entries[i] = {}; - } + _obj = nullptr; + _monitor = nullptr; } #endif // SHARE_RUNTIME_LOCKSTACK_INLINE_HPP