8384807: Populate OMCache from C2

Reviewed-by: aboldtch, coleenp, shade

src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp

@@ -219,6 +219,10 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register t1,
     bind(inflated);
 
     const Register t1_monitor = t1;
+    // Offsets into the current thread's object monitor cache (omc).
+    const ByteSize thr_omc_offset     = JavaThread::om_cache_offset();
+    const ByteSize omc_monitor_offset = OMCache::monitor_offset();
+    const ByteSize omc_obj_offset     = OMCache::obj_offset();
 
     if (!UseObjectMonitorTable) {
       assert(t1_monitor == t1_mark, "should be the same here");
@@ -229,18 +233,12 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register t1,
       // Save the mark, we might need it to extract the hash.
       mov(t3, t1_mark);
 
-      // Look for the monitor in the om_cache.
+      // Look for the monitor in the current thread's object monitor cache (omc).
 
-      ByteSize cache_offset   = JavaThread::om_cache_oops_offset();
-      ByteSize monitor_offset = OMCache::oop_to_monitor_difference();
-      const int num_unrolled  = OMCache::CAPACITY;
-      for (int i = 0; i < num_unrolled; i++) {
-        ldr(t1_monitor, Address(rthread, cache_offset + monitor_offset));
-        ldr(t2, Address(rthread, cache_offset));
-        cmp(obj, t2);
-        br(Assembler::EQ, monitor_found);
-        cache_offset = cache_offset + OMCache::oop_to_oop_difference();
-      }
+      ldr(t1_monitor, Address(rthread, thr_omc_offset + omc_monitor_offset));
+      ldr(t2, Address(rthread, thr_omc_offset + omc_obj_offset));
+      cmp(obj, t2);
+      br(Assembler::EQ, monitor_found);
 
       // Look for the monitor in the table.
 
@@ -268,6 +266,10 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register t1,
       cmp(t3, obj);
       br(Assembler::NE, slow_path);
 
+      // Store the monitor in the current thread's object monitor cache (omc).
+      str(t1_monitor, Address(rthread, thr_omc_offset + omc_monitor_offset));
+      str(obj, Address(rthread, thr_omc_offset + omc_obj_offset));
+
       bind(monitor_found);
     }
 
@@ -296,6 +298,7 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register t1,
 
     bind(monitor_locked);
     if (UseObjectMonitorTable) {
+      // Cache the monitor for unlock.
       str(t1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
     }
   }

src/hotspot/cpu/ppc/macroAssembler_ppc.cpp

@@ -2763,6 +2763,11 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
     const Register monitor    = UseObjectMonitorTable ? tmp1 : noreg;
     const Register owner_addr = tmp2;
     const Register thread_id  = UseObjectMonitorTable ? tmp3 : tmp1;
+    // Offsets into the current thread's object monitor cache (omc).
+    const ByteSize thr_omc_offset     = JavaThread::om_cache_offset();
+    const ByteSize omc_monitor_offset = OMCache::monitor_offset();
+    const ByteSize omc_obj_offset     = OMCache::obj_offset();
+
     Label monitor_locked;
 
     if (!UseObjectMonitorTable) {
@@ -2777,18 +2782,12 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
       // Save the mark, we might need it to extract the hash.
       mr(tmp2_hash, mark);
 
-      // Look for the monitor in the om_cache.
+      // Look for the monitor in the current thread's object monitor cache (omc).
 
-      ByteSize cache_offset   = JavaThread::om_cache_oops_offset();
-      ByteSize monitor_offset = OMCache::oop_to_monitor_difference();
-      const int num_unrolled  = OMCache::CAPACITY;
-      for (int i = 0; i < num_unrolled; i++) {
-        ld(R0, in_bytes(cache_offset), R16_thread);
-        ld(monitor, in_bytes(cache_offset + monitor_offset), R16_thread);
-        cmpd(CR0, R0, obj);
-        beq(CR0, monitor_found);
-        cache_offset = cache_offset + OMCache::oop_to_oop_difference();
-      }
+      ld(R0, in_bytes(thr_omc_offset + omc_obj_offset), R16_thread);
+      ld(monitor, in_bytes(thr_omc_offset + omc_monitor_offset), R16_thread);
+      cmpd(CR0, R0, obj);
+      beq(CR0, monitor_found);
 
       // Look for the monitor in the table.
 
@@ -2817,6 +2816,10 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
       cmpd(CR0, tmp3, obj);
       bne(CR0, slow_path);
 
+      // Store the monitor in the current thread's object monitor cache (omc).
+      std(monitor, in_bytes(thr_omc_offset + omc_monitor_offset), R16_thread);
+      std(obj, in_bytes(thr_omc_offset + omc_obj_offset), R16_thread);
+
       bind(monitor_found);
 
       // Compute owner address.
@@ -2854,6 +2857,7 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
 
     bind(monitor_locked);
     if (UseObjectMonitorTable) {
+      // Cache the monitor for unlock.
       std(monitor, BasicLock::object_monitor_cache_offset_in_bytes(), box);
     }
   }

src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp

@@ -121,6 +121,10 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box,
     bind(inflated);
 
     const Register tmp1_monitor = tmp1;
+    // Offsets into the current thread's object monitor cache (omc).
+    const ByteSize thr_omc_offset     = JavaThread::om_cache_offset();
+    const ByteSize omc_monitor_offset = OMCache::monitor_offset();
+    const ByteSize omc_obj_offset     = OMCache::obj_offset();
 
     if (!UseObjectMonitorTable) {
       assert(tmp1_monitor == tmp1_mark, "should be the same here");
@@ -132,17 +136,11 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box,
       // Save the mark, we might need it to extract the hash.
       mv(tmp2_hash, tmp1_mark);
 
-      // Look for the monitor in the om_cache.
+      // Look for the monitor in the current thread's object monitor cache (omc).
 
-      ByteSize cache_offset   = JavaThread::om_cache_oops_offset();
-      ByteSize monitor_offset = OMCache::oop_to_monitor_difference();
-      const int num_unrolled  = OMCache::CAPACITY;
-      for (int i = 0; i < num_unrolled; i++) {
-        ld(tmp1_monitor, Address(xthread, cache_offset + monitor_offset));
-        ld(tmp4, Address(xthread, cache_offset));
-        beq(obj, tmp4, monitor_found);
-        cache_offset = cache_offset + OMCache::oop_to_oop_difference();
-      }
+      ld(tmp1_monitor, Address(xthread, thr_omc_offset + omc_monitor_offset));
+      ld(tmp4, Address(xthread, thr_omc_offset + omc_obj_offset));
+      beq(obj, tmp4, monitor_found);
 
       // Look for the monitor in the table.
 
@@ -170,6 +168,10 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box,
       bs_asm->try_peek_weak_handle_in_nmethod(this, tmp3, tmp3, tmp2, slow_path);
       bne(tmp3, obj, slow_path);
 
+      // Store the monitor in the current thread's object monitor cache (omc).
+      sd(tmp1_monitor, Address(xthread, thr_omc_offset + omc_monitor_offset));
+      sd(obj, Address(xthread, thr_omc_offset + omc_obj_offset));
+
       bind(monitor_found);
     }
 
@@ -200,6 +202,7 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box,
 
     bind(monitor_locked);
     if (UseObjectMonitorTable) {
+      // Cache the monitor for unlock.
       sd(tmp1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
     }
   }

src/hotspot/cpu/s390/macroAssembler_s390.cpp

@@ -6382,6 +6382,11 @@ void MacroAssembler::compiler_fast_lock_object(Register obj, Register box, Regis
     bind(inflated);
 
     const Register tmp1_monitor = tmp1;
+    // Offsets into the current thread's object monitor cache (omc).
+    const ByteSize thr_omc_offset     = JavaThread::om_cache_offset();
+    const ByteSize omc_monitor_offset = OMCache::monitor_offset();
+    const ByteSize omc_obj_offset     = OMCache::obj_offset();
+
     if (!UseObjectMonitorTable) {
       assert(tmp1_monitor == mark, "should be the same here");
     } else {
@@ -6392,17 +6397,11 @@ void MacroAssembler::compiler_fast_lock_object(Register obj, Register box, Regis
       // Save the mark, we might need it to extract the hash.
       z_lgr(hash, mark);
 
-      // Look for the monitor in the om_cache.
+      // Look for the monitor in the current thread's object monitor cache (omc).
 
-      ByteSize cache_offset   = JavaThread::om_cache_oops_offset();
-      ByteSize monitor_offset = OMCache::oop_to_monitor_difference();
-      const int num_unrolled  = OMCache::CAPACITY;
-      for (int i = 0; i < num_unrolled; i++) {
-        z_lg(tmp1_monitor, Address(Z_thread, cache_offset + monitor_offset));
-        z_cg(obj, Address(Z_thread, cache_offset));
-        z_bre(monitor_found);
-        cache_offset = cache_offset + OMCache::oop_to_oop_difference();
-      }
+      z_lg(tmp1_monitor, Address(Z_thread, thr_omc_offset + omc_monitor_offset));
+      z_cg(obj, Address(Z_thread, thr_omc_offset + omc_obj_offset));
+      z_bre(monitor_found);
 
       // Get the hash code.
       z_srlg(hash, hash, markWord::hash_shift);
@@ -6429,6 +6428,10 @@ void MacroAssembler::compiler_fast_lock_object(Register obj, Register box, Regis
       z_cgr(obj, tmp2);
       z_brne(slow_path);
 
+      // Store the monitor in the current thread's object monitor cache (omc).
+      z_stg(tmp1_monitor, Address(Z_thread, thr_omc_offset + omc_monitor_offset));
+      z_stg(obj, Address(Z_thread, thr_omc_offset + omc_obj_offset));
+
       bind(monitor_found);
     }
     NearLabel monitor_locked;
@@ -6458,7 +6461,7 @@ void MacroAssembler::compiler_fast_lock_object(Register obj, Register box, Regis
 
     bind(monitor_locked);
     if (UseObjectMonitorTable) {
-      // Cache the monitor for unlock
+      // Cache the monitor for unlock.
       z_stg(tmp1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
     }
     // set the CC now

src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp

@@ -294,6 +294,10 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register rax_reg,
     bind(inflated);
 
     const Register monitor = t;
+    // Offsets into the current thread's object monitor cache (omc).
+    const ByteSize thr_omc_offset     = JavaThread::om_cache_offset();
+    const ByteSize omc_monitor_offset = OMCache::monitor_offset();
+    const ByteSize omc_obj_offset     = OMCache::obj_offset();
 
     if (!UseObjectMonitorTable) {
       assert(mark == monitor, "should be the same here");
@@ -301,17 +305,11 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register rax_reg,
       const Register hash = t;
       Label monitor_found;
 
-      // Look for the monitor in the om_cache.
+      // Look for the monitor in the current thread's object monitor cache (omc).
 
-      ByteSize cache_offset   = JavaThread::om_cache_oops_offset();
-      ByteSize monitor_offset = OMCache::oop_to_monitor_difference();
-      const int num_unrolled  = OMCache::CAPACITY;
-      for (int i = 0; i < num_unrolled; i++) {
-        movptr(monitor, Address(thread,  cache_offset + monitor_offset));
-        cmpptr(obj, Address(thread, cache_offset));
-        jccb(Assembler::equal, monitor_found);
-        cache_offset = cache_offset + OMCache::oop_to_oop_difference();
-      }
+      movptr(monitor, Address(thread, thr_omc_offset + omc_monitor_offset));
+      cmpptr(obj, Address(thread, thr_omc_offset + omc_obj_offset));
+      jccb(Assembler::equal, monitor_found);
 
       // Look for the monitor in the table.
 
@@ -340,6 +338,10 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register rax_reg,
       cmpptr(rax_reg, obj);
       jcc(Assembler::notEqual, slow_path);
 
+      // Store the monitor in the current thread's object monitor cache (omc).
+      movptr(Address(thread, thr_omc_offset + omc_monitor_offset), monitor);
+      movptr(Address(thread, thr_omc_offset + omc_obj_offset), obj);
+
       bind(monitor_found);
     }
     const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast<int>(markWord::monitor_value));

src/hotspot/share/runtime/javaThread.hpp

@@ -1238,7 +1238,6 @@ public:
   static ByteSize lock_stack_base_offset() { return lock_stack_offset() + LockStack::base_offset(); }
 
   static ByteSize om_cache_offset()        { return byte_offset_of(JavaThread, _om_cache); }
-  static ByteSize om_cache_oops_offset()   { return om_cache_offset() + OMCache::entries_offset(); }
 
   void om_set_monitor_cache(ObjectMonitor* monitor);
   void om_clear_monitor_cache();

src/hotspot/share/runtime/lockStack.cpp

@@ -115,7 +115,6 @@ void LockStack::print_on(outputStream* st) {
   }
 }
 
-OMCache::OMCache(JavaThread* jt) : _entries() {
+OMCache::OMCache(JavaThread* jt) {
   STATIC_ASSERT(std::is_standard_layout<OMCache>::value);
-  STATIC_ASSERT(std::is_standard_layout<OMCache::OMCacheEntry>::value);
 }

src/hotspot/share/runtime/lockStack.hpp

@@ -130,19 +130,14 @@ class LockStack {
 
 class OMCache {
   friend class VMStructs;
- public:
-  static constexpr int CAPACITY = 2;
 
  private:
-  struct OMCacheEntry {
-    oop _oop = nullptr;
-    ObjectMonitor* _monitor = nullptr;
-  } _entries[CAPACITY];
+  oop _obj = nullptr;
+  ObjectMonitor* _monitor = nullptr;
 
  public:
-  static ByteSize entries_offset() { return byte_offset_of(OMCache, _entries); }
-  static constexpr ByteSize oop_to_oop_difference() { return in_ByteSize(sizeof(OMCacheEntry)); }
-  static constexpr ByteSize oop_to_monitor_difference() { return in_ByteSize(sizeof(oop)); }
+  static constexpr ByteSize obj_offset() { return byte_offset_of(OMCache, _obj); }
+  static constexpr ByteSize monitor_offset() { return byte_offset_of(OMCache, _monitor); }
 
   explicit OMCache(JavaThread* jt);
 

src/hotspot/share/runtime/lockStack.inline.hpp

@@ -250,53 +250,30 @@ inline void LockStack::oops_do(OopClosure* cl) {
 }
 
 inline void OMCache::set_monitor(ObjectMonitor *monitor) {
-  const int end = OMCache::CAPACITY - 1;
-
   oop obj = monitor->object_peek();
   assert(obj != nullptr, "must be alive");
   assert(monitor == ObjectSynchronizer::get_monitor_from_table(obj), "must exist in table");
 
-  OMCacheEntry to_insert = {obj, monitor};
-
-  for (int i = 0; i < end; ++i) {
-    if (_entries[i]._oop == obj ||
-        _entries[i]._monitor == nullptr ||
-        _entries[i]._monitor->is_being_async_deflated()) {
-      // Use stale slot.
-      _entries[i] = to_insert;
-      return;
-    }
-    // Swap with the most recent value.
-    ::swap(to_insert, _entries[i]);
-  }
-  _entries[end] = to_insert;
+  _monitor = monitor;
+  _obj     = obj;
 }
 
 inline ObjectMonitor* OMCache::get_monitor(oop o) {
-  for (int i = 0; i < CAPACITY; ++i) {
-    if (_entries[i]._oop == o) {
-      assert(_entries[i]._monitor != nullptr, "monitor must exist");
-      if (_entries[i]._monitor->is_being_async_deflated()) {
-        // Bad monitor
-        // Shift down rest
-        for (; i < CAPACITY - 1; ++i) {
-          _entries[i] = _entries[i + 1];
-        }
-        // Clear end
-        _entries[i] = {};
-        return nullptr;
-      }
-      return _entries[i]._monitor;
+  if (_obj == o) {
+    assert(_monitor != nullptr, "monitor must exist");
+    if (!_monitor->is_being_async_deflated()) {
+      return _monitor;
     }
+    // Bad monitor, so clear the cache.
+    _obj     = nullptr;
+    _monitor = nullptr;
   }
   return nullptr;
 }
 
 inline void OMCache::clear() {
-  for (size_t i = 0; i < CAPACITY; ++i) {
-    // Clear
-    _entries[i] = {};
-  }
+  _obj     = nullptr;
+  _monitor = nullptr;
 }
 
 #endif // SHARE_RUNTIME_LOCKSTACK_INLINE_HPP