< prev index next >

src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp

Print this page
*** 947,10 ***
--- 947,13 ---
    // Finish fast lock successfully. ZF value is irrelevant.
    Label locked;
    // Finish fast lock unsuccessfully. MUST jump with ZF == 0
    Label slow_path;
  
+   // Clear box. TODO[OMWorld]: Is this necessary? May also defer this to not write twice.
+   movptr(Address(box, BasicLock::displaced_header_offset_in_bytes()), 0);
+ 
    if (DiagnoseSyncOnValueBasedClasses != 0) {
      load_klass(rax_reg, obj, t);
      movl(rax_reg, Address(rax_reg, Klass::access_flags_offset()));
      testl(rax_reg, JVM_ACC_IS_VALUE_BASED_CLASS);
      jcc(Assembler::notZero, slow_path);

*** 960,11 ***
  
    { // Lightweight Lock
  
      Label push;
  
!     const Register top = box;
  
      // Load the mark.
      movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
  
      // Prefetch top.
--- 963,11 ---
  
    { // Lightweight Lock
  
      Label push;
  
!     const Register top = rax_reg;
  
      // Load the mark.
      movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
  
      // Prefetch top.

*** 989,31 ***
      lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
      jcc(Assembler::notEqual, slow_path);
  
      bind(push);
      // After successful lock, push object on lock-stack.
      movptr(Address(thread, top), obj);
      addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
      jmpb(locked);
    }
  
    { // Handle inflated monitor.
      bind(inflated);
  
!     const Register tagged_monitor = mark;
  
!     // CAS owner (null => current thread).
!     xorptr(rax_reg, rax_reg);
-     lock(); cmpxchgptr(thread, Address(tagged_monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
-     jccb(Assembler::equal, locked);
  
!     // Check if recursive.
!     cmpptr(thread, rax_reg);
!     jccb(Assembler::notEqual, slow_path);
  
!     // Recursive.
!     increment(Address(tagged_monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
    }
  
    bind(locked);
    increment(Address(thread, JavaThread::held_monitor_count_offset()));
    // Set ZF = 1
--- 992,102 ---
      lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
      jcc(Assembler::notEqual, slow_path);
  
      bind(push);
      // After successful lock, push object on lock-stack.
+     // TODO[OMWorld]: Was prepush better?
+     movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
      movptr(Address(thread, top), obj);
      addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
      jmpb(locked);
    }
  
    { // Handle inflated monitor.
      bind(inflated);
  
!     const Register monitor = t;
+ 
+     if (!OMUseC2Cache) {
+       jmp(slow_path);
+     } else {
+       if (OMCacheHitRate) increment(Address(thread, JavaThread::lock_lookup_offset()));
  
!       // Fetch ObjectMonitor* from the cache or take the slow-path.
!       Label monitor_found;
  
!       // Load cache address
!       lea(t, Address(thread, JavaThread::om_cache_oops_offset()));
! 
+       const int num_unrolled = MIN2(OMC2UnrollCacheEntries, OMCacheSize);
+       for (int i = 0; i < num_unrolled; i++) {
+         cmpptr(obj, Address(t));
+         jccb(Assembler::equal, monitor_found);
+         if (i + 1 != num_unrolled) {
+           increment(t, in_bytes(OMCache::oop_to_oop_difference()));
+         }
+       }
+ 
+       if (num_unrolled == 0 || (OMC2UnrollCacheLookupLoopTail && num_unrolled != OMCacheSize)) {
+         if (num_unrolled != 0) {
+           // Loop after unrolling, advance iterator.
+           increment(t, in_bytes(OMCache::oop_to_oop_difference()));
+         }
+ 
+         Label loop;
  
!         // Search for obj in cache.
!         bind(loop);
+ 
+         // Check for match.
+         cmpptr(obj, Address(t));
+         jccb(Assembler::equal, monitor_found);
+ 
+         // Search until null encountered, guaranteed _null_sentinel at end.
+         cmpptr(Address(t), 1);
+         jcc(Assembler::below, slow_path); // 0 check, but with ZF=0 when *t == 0
+         increment(t, in_bytes(OMCache::oop_to_oop_difference()));
+         jmpb(loop);
+       } else {
+         jmp(slow_path);
+       }
+ 
+       // Cache hit.
+       bind(monitor_found);
+       movptr(monitor, Address(t, OMCache::oop_to_monitor_difference()));
+       if (OMCacheHitRate) increment(Address(thread, JavaThread::lock_hit_offset()));
+ 
+       Label monitor_locked;
+       // Lock the monitor.
+       Label recursion;
+       if (OMRecursiveFastPath) {
+         // Check owner for recursion first.
+         cmpptr(thread, Address(monitor, ObjectMonitor::owner_offset()));
+         jccb(Assembler::equal, recursion);
+       }
+ 
+       // CAS owner (null => current thread).
+       xorptr(rax, rax);
+       lock(); cmpxchgptr(thread, Address(monitor, ObjectMonitor::owner_offset()));
+       jccb(Assembler::equal, monitor_locked);
+ 
+       if (OMRecursiveFastPath) {
+         // Recursion already checked.
+         jmpb(slow_path);
+       } else {
+         // Check if recursive.
+         cmpptr(thread, rax);
+         jccb(Assembler::notEqual, slow_path);
+       }
+ 
+       // Recursive.
+       bind(recursion);
+       increment(Address(monitor, ObjectMonitor::recursions_offset()));
+ 
+       bind(monitor_locked);
+       // Cache the monitor for unlock
+       movptr(Address(box, BasicLock::displaced_header_offset_in_bytes()), monitor);
+     }
    }
  
    bind(locked);
    increment(Address(thread, JavaThread::held_monitor_count_offset()));
    // Set ZF = 1

*** 1051,11 ***
  
    // Assume success.
    decrement(Address(thread, JavaThread::held_monitor_count_offset()));
  
    const Register mark = t;
!   const Register top = reg_rax;
  
    Label dummy;
    C2FastUnlockLightweightStub* stub = nullptr;
  
    if (!Compile::current()->output()->in_scratch_emit_size()) {
--- 1125,13 ---
  
    // Assume success.
    decrement(Address(thread, JavaThread::held_monitor_count_offset()));
  
    const Register mark = t;
!   const Register monitor = t;
+   const Register top = t;
+   const Register box = reg_rax;
  
    Label dummy;
    C2FastUnlockLightweightStub* stub = nullptr;
  
    if (!Compile::current()->output()->in_scratch_emit_size()) {

*** 1063,19 ***
      Compile::current()->output()->add_stub(stub);
    }
  
    Label& push_and_slow_path = stub == nullptr ? dummy : stub->push_and_slow_path();
    Label& check_successor = stub == nullptr ? dummy : stub->check_successor();
  
    { // Lightweight Unlock
  
      // Load top.
      movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
  
-     // Prefetch mark.
-     movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
- 
      // Check if obj is top of lock-stack.
      cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
      // Top of lock stack was not obj. Must be monitor.
      jcc(Assembler::notEqual, inflated_check_lock_stack);
  
--- 1139,17 ---
      Compile::current()->output()->add_stub(stub);
    }
  
    Label& push_and_slow_path = stub == nullptr ? dummy : stub->push_and_slow_path();
    Label& check_successor = stub == nullptr ? dummy : stub->check_successor();
+   Label& slow_path = stub == nullptr ? dummy : stub->slow_path();
  
    { // Lightweight Unlock
  
      // Load top.
      movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
  
      // Check if obj is top of lock-stack.
      cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
      // Top of lock stack was not obj. Must be monitor.
      jcc(Assembler::notEqual, inflated_check_lock_stack);
  

*** 1087,10 ***
--- 1161,13 ---
      cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
      jcc(Assembler::equal, unlocked);
  
      // We elide the monitor check, let the CAS fail instead.
  
+     // Load mark.
+     movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+ 
      // Try to unlock. Transition lock bits 0b00 => 0b01
      movptr(reg_rax, mark);
      andptr(reg_rax, ~(int32_t)markWord::lock_mask);
      orptr(mark, markWord::unlocked_value);
      lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));

*** 1108,54 ***
      jcc(Assembler::below, check_done);
      cmpptr(obj, Address(thread, top));
      jccb(Assembler::notEqual, inflated_check_lock_stack);
      stop("Fast Unlock lock on stack");
      bind(check_done);
      testptr(mark, markWord::monitor_value);
      jccb(Assembler::notZero, inflated);
      stop("Fast Unlock not monitor");
  #endif
  
      bind(inflated);
  
!     // mark contains the tagged ObjectMonitor*.
!     const Register monitor = mark;
! 
  #ifndef _LP64
!     // Check if recursive.
!     xorptr(reg_rax, reg_rax);
!     orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
!     jcc(Assembler::notZero, check_successor);
  
!     // Check if the entry lists are empty.
!     movptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
!     orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
-     jcc(Assembler::notZero, check_successor);
  
!     // Release lock.
!     movptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
! #else // _LP64
!     Label recursive;
  
!     // Check if recursive.
!     cmpptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)), 0);
!     jccb(Assembler::notEqual, recursive);
! 
!     // Check if the entry lists are empty.
!     movptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
!     orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
!     jcc(Assembler::notZero, check_successor);
- 
-     // Release lock.
-     movptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
-     jmpb(unlocked);
- 
-     // Recursive unlock.
-     bind(recursive);
-     decrement(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
-     xorl(t, t);
  #endif
    }
  
    bind(unlocked);
    if (stub != nullptr) {
      bind(stub->unlocked_continuation());
--- 1185,67 ---
      jcc(Assembler::below, check_done);
      cmpptr(obj, Address(thread, top));
      jccb(Assembler::notEqual, inflated_check_lock_stack);
      stop("Fast Unlock lock on stack");
      bind(check_done);
+     const Register mark = t;
+     movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
      testptr(mark, markWord::monitor_value);
      jccb(Assembler::notZero, inflated);
      stop("Fast Unlock not monitor");
  #endif
  
      bind(inflated);
  
!     if (!OMUseC2Cache) {
!       jmp(slow_path);
!     } else {
+       if (OMCacheHitRate) increment(Address(thread, JavaThread::unlock_lookup_offset()));
+       movptr(monitor, Address(box, BasicLock::displaced_header_offset_in_bytes()));
+       // TODO[OMWorld]: Figure out the correctness surrounding the owner field here. Obj is not on the lock stack
+       //                but this means this thread must have locked on the inflated monitor at some point. So it
+       //                should not be anonymous.
+       cmpptr(monitor, 2);
+       jcc(Assembler::below, slow_path);
+ 
+       if (OMCacheHitRate) increment(Address(thread, JavaThread::unlock_hit_offset()));
  #ifndef _LP64
!         // TODO[OMWorld]: Unify 32 with 64. Should just be a straight up use 64 on 32. We have the registers here.
!         // Check if recursive.
!         xorptr(reg_rax, reg_rax);
!         orptr(reg_rax, Address(monitor, ObjectMonitor::recursions_offset()));
+         jcc(Assembler::notZero, check_successor);
+ 
+         // Check if the entry lists are empty.
+         movptr(reg_rax, Address(monitor, ObjectMonitor::EntryList_offset()));
+         orptr(reg_rax, Address(monitor, ObjectMonitor::cxq_offset()));
+         jcc(Assembler::notZero, check_successor);
+ 
+         // Release lock.
+         movptr(Address(monitor, ObjectMonitor::owner_offset()), NULL_WORD);
+ #else // _LP64
+         Label recursive;
  
!         // Check if recursive.
!         cmpptr(Address(monitor,ObjectMonitor::recursions_offset()),0);
!         jccb(Assembler::notEqual, recursive);
  
!         // Check if the entry lists are empty.
!         movptr(reg_rax, Address(monitor, ObjectMonitor::cxq_offset()));
!         orptr(reg_rax, Address(monitor, ObjectMonitor::EntryList_offset()));
!         jcc(Assembler::notZero, check_successor);
  
!         // Release lock.
!         movptr(Address(monitor, ObjectMonitor::owner_offset()), NULL_WORD);
!         jmpb(unlocked);
! 
!         // Recursive unlock.
!         bind(recursive);
!         decrement(Address(monitor, ObjectMonitor::recursions_offset()));
!         xorl(t, t);
  #endif
+     }
    }
  
    bind(unlocked);
    if (stub != nullptr) {
      bind(stub->unlocked_continuation());

*** 6495,5 ***
--- 6585,21 ---
    } else {
      assert(bt == T_FLOAT, "");
      vpermps(dst, shuffle, src, vlen_enc);
    }
  }
+ 
+ #ifdef _LP64
+ void C2_MacroAssembler::load_nklass_compact_c2(Register dst, Register obj, Register index, Address::ScaleFactor scale, int disp) {
+   // Note: Don't clobber obj anywhere in that method!
+ 
+   // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
+   // obj-start, so that we can load from the object's mark-word instead. Usually the address
+   // comes as obj-start in obj and klass_offset_in_bytes in disp. However, sometimes C2
+   // emits code that pre-computes obj-start + klass_offset_in_bytes into a register, and
+   // then passes that register as obj and 0 in disp. The following code extracts the base
+   // and offset to load the mark-word.
+   int offset = oopDesc::mark_offset_in_bytes() + disp - oopDesc::klass_offset_in_bytes();
+   movq(dst, Address(obj, index, scale, offset));
+   shrq(dst, markWord::klass_shift);
+ }
+ #endif
< prev index next >