< prev index next >

src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp

Print this page
@@ -947,10 +947,15 @@
    // Finish fast lock successfully. ZF value is irrelevant.
    Label locked;
    // Finish fast lock unsuccessfully. MUST jump with ZF == 0
    Label slow_path;
  
+   if (UseObjectMonitorTable) {
+     // Clear cache in case fast locking succeeds.
+     movptr(Address(box, BasicLock::object_monitor_cache_offset_in_bytes()), 0);
+   }
+ 
    if (DiagnoseSyncOnValueBasedClasses != 0) {
      load_klass(rax_reg, obj, t);
      movl(rax_reg, Address(rax_reg, Klass::access_flags_offset()));
      testl(rax_reg, JVM_ACC_IS_VALUE_BASED_CLASS);
      jcc(Assembler::notZero, slow_path);

@@ -960,11 +965,11 @@
  
    { // Lightweight Lock
  
      Label push;
  
-     const Register top = box;
+     const Register top = UseObjectMonitorTable ? rax_reg : box;
  
      // Load the mark.
      movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
  
      // Prefetch top.

@@ -987,33 +992,100 @@
      orptr(rax_reg, markWord::unlocked_value);
      andptr(mark, ~(int32_t)markWord::unlocked_value);
      lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
      jcc(Assembler::notEqual, slow_path);
  
+     if (UseObjectMonitorTable) {
+       // Need to reload top, clobbered by CAS.
+       movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
+     }
      bind(push);
      // After successful lock, push object on lock-stack.
      movptr(Address(thread, top), obj);
      addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
      jmpb(locked);
    }
  
    { // Handle inflated monitor.
      bind(inflated);
  
-     const Register tagged_monitor = mark;
+     const Register monitor = t;
+ 
+     if (!UseObjectMonitorTable) {
+       assert(mark == monitor, "should be the same here");
+     } else {
+       // Uses ObjectMonitorTable.  Look for the monitor in the om_cache.
+       if (OMCacheHitRate) increment(Address(thread, JavaThread::lock_lookup_offset()));
+ 
+       // Fetch ObjectMonitor* from the cache or take the slow-path.
+       Label monitor_found;
+ 
+       // Load cache address
+       lea(t, Address(thread, JavaThread::om_cache_oops_offset()));
+ 
+       const int num_unrolled = MIN2(OMC2UnrollCacheEntries, OMCacheSize);
+       for (int i = 0; i < num_unrolled; i++) {
+         cmpptr(obj, Address(t));
+         jccb(Assembler::equal, monitor_found);
+         if (i + 1 != num_unrolled) {
+           increment(t, in_bytes(OMCache::oop_to_oop_difference()));
+         }
+       }
+ 
+       if (num_unrolled == 0 || (OMC2UnrollCacheLookupLoopTail && num_unrolled != OMCacheSize)) {
+         if (num_unrolled != 0) {
+           // Loop after unrolling, advance iterator.
+           increment(t, in_bytes(OMCache::oop_to_oop_difference()));
+         }
+ 
+         Label loop;
+ 
+         // Search for obj in cache.
+         bind(loop);
+ 
+         // Check for match.
+         cmpptr(obj, Address(t));
+         jccb(Assembler::equal, monitor_found);
+ 
+         // Search until null encountered, guaranteed _null_sentinel at end.
+         cmpptr(Address(t), 1);
+         jcc(Assembler::below, slow_path); // 0 check, but with ZF=0 when *t == 0
+         increment(t, in_bytes(OMCache::oop_to_oop_difference()));
+         jmpb(loop);
+       } else {
+         jmp(slow_path);
+       }
+ 
+       // Cache hit.
+       bind(monitor_found);
+       movptr(monitor, Address(t, OMCache::oop_to_monitor_difference()));
+       if (OMCacheHitRate) increment(Address(thread, JavaThread::lock_hit_offset()));
+     }
+     const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast<int>(markWord::monitor_value));
+     const Address recursions_address{monitor, ObjectMonitor::recursions_offset() - monitor_tag};
+     const Address owner_address{monitor, ObjectMonitor::owner_offset() - monitor_tag};
+ 
+     Label monitor_locked;
+     // Lock the monitor.
  
      // CAS owner (null => current thread).
      xorptr(rax_reg, rax_reg);
-     lock(); cmpxchgptr(thread, Address(tagged_monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
-     jccb(Assembler::equal, locked);
+     lock(); cmpxchgptr(thread, owner_address);
+     jccb(Assembler::equal, monitor_locked);
  
      // Check if recursive.
      cmpptr(thread, rax_reg);
      jccb(Assembler::notEqual, slow_path);
  
      // Recursive.
-     increment(Address(tagged_monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
+     increment(recursions_address);
+ 
+     bind(monitor_locked);
+     if (UseObjectMonitorTable) {
+       // Cache the monitor for unlock
+       movptr(Address(box, BasicLock::object_monitor_cache_offset_in_bytes()), monitor);
+     }
    }
  
    bind(locked);
    increment(Address(thread, JavaThread::held_monitor_count_offset()));
    // Set ZF = 1

@@ -1051,11 +1123,13 @@
  
    // Assume success.
    decrement(Address(thread, JavaThread::held_monitor_count_offset()));
  
    const Register mark = t;
-   const Register top = reg_rax;
+   const Register monitor = t;
+   const Register top = UseObjectMonitorTable ? t : reg_rax;
+   const Register box = reg_rax;
  
    Label dummy;
    C2FastUnlockLightweightStub* stub = nullptr;
  
    if (!Compile::current()->output()->in_scratch_emit_size()) {

@@ -1063,18 +1137,21 @@
      Compile::current()->output()->add_stub(stub);
    }
  
    Label& push_and_slow_path = stub == nullptr ? dummy : stub->push_and_slow_path();
    Label& check_successor = stub == nullptr ? dummy : stub->check_successor();
+   Label& slow_path = stub == nullptr ? dummy : stub->slow_path();
  
    { // Lightweight Unlock
  
      // Load top.
      movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
  
-     // Prefetch mark.
-     movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+     if (!UseObjectMonitorTable) {
+       // Prefetch mark.
+       movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+     }
  
      // Check if obj is top of lock-stack.
      cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
      // Top of lock stack was not obj. Must be monitor.
      jcc(Assembler::notEqual, inflated_check_lock_stack);

@@ -1087,10 +1164,15 @@
      cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
      jcc(Assembler::equal, unlocked);
  
      // We elide the monitor check, let the CAS fail instead.
  
+     if (UseObjectMonitorTable) {
+       // Load mark.
+       movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+     }
+ 
      // Try to unlock. Transition lock bits 0b00 => 0b01
      movptr(reg_rax, mark);
      andptr(reg_rax, ~(int32_t)markWord::lock_mask);
      orptr(mark, markWord::unlocked_value);
      lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));

@@ -1108,54 +1190,57 @@
      jcc(Assembler::below, check_done);
      cmpptr(obj, Address(thread, top));
      jccb(Assembler::notEqual, inflated_check_lock_stack);
      stop("Fast Unlock lock on stack");
      bind(check_done);
+     if (UseObjectMonitorTable) {
+       movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+     }
      testptr(mark, markWord::monitor_value);
      jccb(Assembler::notZero, inflated);
      stop("Fast Unlock not monitor");
  #endif
  
      bind(inflated);
  
-     // mark contains the tagged ObjectMonitor*.
-     const Register monitor = mark;
- 
- #ifndef _LP64
-     // Check if recursive.
-     xorptr(reg_rax, reg_rax);
-     orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
-     jcc(Assembler::notZero, check_successor);
- 
-     // Check if the entry lists are empty.
-     movptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
-     orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
-     jcc(Assembler::notZero, check_successor);
+     if (!UseObjectMonitorTable) {
+       assert(mark == monitor, "should be the same here");
+     } else {
+       // Uses ObjectMonitorTable.  Look for the monitor in our BasicLock on the stack.
+       if (OMCacheHitRate) increment(Address(thread, JavaThread::unlock_lookup_offset()));
+       movptr(monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
+       // null check with ZF == 0, no valid pointer below alignof(ObjectMonitor*)
+       cmpptr(monitor, alignof(ObjectMonitor*));
+       jcc(Assembler::below, slow_path);
+ 
+       if (OMCacheHitRate) increment(Address(thread, JavaThread::unlock_hit_offset()));
+     }
+     const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast<int>(markWord::monitor_value));
+     const Address recursions_address{monitor, ObjectMonitor::recursions_offset() - monitor_tag};
+     const Address cxq_address{monitor, ObjectMonitor::cxq_offset() - monitor_tag};
+     const Address EntryList_address{monitor, ObjectMonitor::EntryList_offset() - monitor_tag};
+     const Address owner_address{monitor, ObjectMonitor::owner_offset() - monitor_tag};
  
-     // Release lock.
-     movptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
- #else // _LP64
      Label recursive;
  
      // Check if recursive.
-     cmpptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)), 0);
+     cmpptr(recursions_address, 0);
      jccb(Assembler::notEqual, recursive);
  
      // Check if the entry lists are empty.
-     movptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
-     orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
+     movptr(reg_rax, cxq_address);
+     orptr(reg_rax, EntryList_address);
      jcc(Assembler::notZero, check_successor);
  
      // Release lock.
-     movptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
+     movptr(owner_address, NULL_WORD);
      jmpb(unlocked);
  
      // Recursive unlock.
      bind(recursive);
-     decrement(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
+     decrement(recursions_address);
      xorl(t, t);
- #endif
    }
  
    bind(unlocked);
    if (stub != nullptr) {
      bind(stub->unlocked_continuation());

@@ -6619,5 +6704,21 @@
    } else {
      assert(bt == T_FLOAT, "");
      vpermps(dst, shuffle, src, vlen_enc);
    }
  }
+ 
+ #ifdef _LP64
+ void C2_MacroAssembler::load_nklass_compact_c2(Register dst, Register obj, Register index, Address::ScaleFactor scale, int disp) {
+   // Note: Don't clobber obj anywhere in that method!
+ 
+   // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
+   // obj-start, so that we can load from the object's mark-word instead. Usually the address
+   // comes as obj-start in obj and klass_offset_in_bytes in disp. However, sometimes C2
+   // emits code that pre-computes obj-start + klass_offset_in_bytes into a register, and
+   // then passes that register as obj and 0 in disp. The following code extracts the base
+   // and offset to load the mark-word.
+   int offset = oopDesc::mark_offset_in_bytes() + disp - oopDesc::klass_offset_in_bytes();
+   movq(dst, Address(obj, index, scale, offset));
+   shrq(dst, markWord::klass_shift);
+ }
+ #endif
< prev index next >