< prev index next >

src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp

Print this page
@@ -457,22 +457,25 @@
    Register threadReg = r15_thread;
  #else
    get_thread(scrReg);
    Register threadReg = scrReg;
  #endif
+   movptr(scrReg, Address(threadReg, JavaThread::lock_id_offset()));
    lock();
-   cmpxchgptr(threadReg, Address(boxReg, owner_offset)); // Updates tmpReg
+   cmpxchgptr(scrReg, Address(boxReg, owner_offset)); // Updates tmpReg
  
    if (RTMRetryCount > 0) {
      // success done else retry
      jccb(Assembler::equal, DONE_LABEL) ;
      bind(L_decrement_retry);
      // Spin and retry if lock is busy.
      rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry);
+     jmp(DONE_LABEL);
    }
    else {
      bind(L_decrement_retry);
+     jmp(DONE_LABEL);
    }
  }
  
  #endif //  INCLUDE_RTM_OPT
  

@@ -583,11 +586,11 @@
    //       = sp-proximity test hits
    //       = sp-proximity test generates false-negative
    //    -- by other
    //
  
-   Label IsInflated, DONE_LABEL, NO_COUNT, COUNT;
+   Label IsInflated, DONE_LABEL, COUNT;
  
    if (DiagnoseSyncOnValueBasedClasses != 0) {
      load_klass(tmpReg, objReg, scrReg);
      movl(tmpReg, Address(tmpReg, Klass::access_flags_offset()));
      testl(tmpReg, JVM_ACC_IS_VALUE_BASED_CLASS);

@@ -625,10 +628,11 @@
      subptr(tmpReg, rsp);
      // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
      andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - (int)os::vm_page_size())) );
      movptr(Address(boxReg, 0), tmpReg);
    }
+   // After recursive stack locking attempt case
    jmp(DONE_LABEL);
  
    bind(IsInflated);
    // The object is inflated. tmpReg contains pointer to ObjectMonitor* + markWord::monitor_value
  

@@ -657,69 +661,69 @@
    // Optimistic form: consider XORL tmpReg,tmpReg
    movptr(tmpReg, NULL_WORD);
  
    // Appears unlocked - try to swing _owner from null to non-null.
    // Ideally, I'd manifest "Self" with get_thread and then attempt
-   // to CAS the register containing Self into m->Owner.
+   // to CAS the register containing thread id into m->Owner.
    // But we don't have enough registers, so instead we can either try to CAS
    // rsp or the address of the box (in scr) into &m->owner.  If the CAS succeeds
-   // we later store "Self" into m->Owner.  Transiently storing a stack address
+   // we later store thread id into m->Owner.  Transiently storing a stack address
    // (rsp or the address of the box) into  m->owner is harmless.
    // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
    lock();
    cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
    movptr(Address(scrReg, 0), 3);          // box->_displaced_header = 3
    // If we weren't able to swing _owner from null to the BasicLock
    // then take the slow path.
-   jccb  (Assembler::notZero, NO_COUNT);
+   jccb  (Assembler::notZero, DONE_LABEL);
    // update _owner from BasicLock to thread
    get_thread (scrReg);                    // beware: clobbers ICCs
+   movptr(scrReg, Address(scrReg, JavaThread::lock_id_offset()));
    movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg);
    xorptr(boxReg, boxReg);                 // set icc.ZFlag = 1 to indicate success
+   jmp(DONE_LABEL);
  
    // If the CAS fails we can either retry or pass control to the slow path.
    // We use the latter tactic.
    // Pass the CAS result in the icc.ZFlag into DONE_LABEL
    // If the CAS was successful ...
    //   Self has acquired the lock
    //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
    // Intentional fall-through into DONE_LABEL ...
  #else // _LP64
+   // Unconditionally set box->_displaced_header = markWord::unused_mark().
+   // Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
+   movptr(Address(boxReg, 0), checked_cast<int32_t>(markWord::unused_mark().value()));
+ 
    // It's inflated and we use scrReg for ObjectMonitor* in this section.
    movq(scrReg, tmpReg);
    xorq(tmpReg, tmpReg);
+   movptr(boxReg, Address(r15_thread, JavaThread::lock_id_offset()));
    lock();
-   cmpxchgptr(thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
-   // Unconditionally set box->_displaced_header = markWord::unused_mark().
-   // Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
-   movptr(Address(boxReg, 0), checked_cast<int32_t>(markWord::unused_mark().value()));
+   cmpxchgptr(boxReg, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
+ 
    // Propagate ICC.ZF from CAS above into DONE_LABEL.
-   jccb(Assembler::equal, COUNT);          // CAS above succeeded; propagate ZF = 1 (success)
+   jccb(Assembler::equal, DONE_LABEL);    // CAS above succeeded; propagate ZF = 1 (success)
  
-   cmpptr(thread, rax);                // Check if we are already the owner (recursive lock)
-   jccb(Assembler::notEqual, NO_COUNT);    // If not recursive, ZF = 0 at this point (fail)
+   cmpptr(boxReg, rax);                // Check if we are already the owner (recursive lock)
+   jccb(Assembler::notEqual, DONE_LABEL);    // If not recursive, ZF = 0 at this point (fail)
    incq(Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
    xorq(rax, rax); // Set ZF = 1 (success) for recursive lock, denoting locking success
+   jmp(DONE_LABEL);
  #endif // _LP64
  #if INCLUDE_RTM_OPT
    } // use_rtm()
  #endif
-   bind(DONE_LABEL);
- 
-   // ZFlag == 1 count in fast path
-   // ZFlag == 0 count in slow path
-   jccb(Assembler::notZero, NO_COUNT); // jump if ZFlag == 0
  
    bind(COUNT);
    // Count monitors in fast path
    increment(Address(thread, JavaThread::held_monitor_count_offset()));
- 
    xorl(tmpReg, tmpReg); // Set ZF == 1
  
-   bind(NO_COUNT);
+   bind(DONE_LABEL);
  
-   // At NO_COUNT the icc ZFlag is set as follows ...
+   // At DONE_LABEL the icc ZFlag is set as follows ...
    // fast_unlock uses the same protocol.
    // ZFlag == 1 -> Success
    // ZFlag == 0 -> Failure - force control through the slow path
  }
  

@@ -753,16 +757,16 @@
  // could reasonably *avoid* checking owner in fast_unlock().
  // In the interest of performance we elide m->Owner==Self check in unlock.
  // A perfectly viable alternative is to elide the owner check except when
  // Xcheck:jni is enabled.
  
- void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, bool use_rtm) {
+ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, Register scrReg, bool use_rtm) {
    assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_unlock_lightweight");
    assert(boxReg == rax, "");
    assert_different_registers(objReg, boxReg, tmpReg);
  
-   Label DONE_LABEL, Stacked, COUNT, NO_COUNT;
+   Label DONE_LABEL, Stacked, COUNT;
  
  #if INCLUDE_RTM_OPT
    if (UseRTMForStackLocks && use_rtm) {
      assert(LockingMode != LM_MONITOR, "LockingMode == 0 (LM_MONITOR) and +UseRTMForStackLocks are mutually exclusive");
      Label L_regular_unlock;

@@ -776,19 +780,34 @@
    }
  #endif
  
    if (LockingMode == LM_LEGACY) {
      cmpptr(Address(boxReg, 0), NULL_WORD);                            // Examine the displaced header
-     jcc   (Assembler::zero, COUNT);                                   // 0 indicates recursive stack-lock
+     jcc   (Assembler::zero, DONE_LABEL);                              // 0 indicates recursive stack-lock
    }
    movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));   // Examine the object's markword
    if (LockingMode != LM_MONITOR) {
      testptr(tmpReg, markWord::monitor_value);                         // Inflated?
      jcc(Assembler::zero, Stacked);
    }
  
    // It's inflated.
+   // If the owner is ANONYMOUS, we need to fix it -  in an outline stub.
+   cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t) ObjectMonitor::ANONYMOUS_OWNER);
+ #ifdef _LP64
+   if (!Compile::current()->output()->in_scratch_emit_size()) {
+     C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmpReg, boxReg);
+     Compile::current()->output()->add_stub(stub);
+     jcc(Assembler::equal, stub->entry());
+     bind(stub->continuation());
+   } else
+ #endif
+   {
+     // We can't easily implement this optimization on 32 bit because we don't have a thread register.
+     // Call the slow-path instead.
+     jcc(Assembler::notEqual, DONE_LABEL);
+   }
  
  #if INCLUDE_RTM_OPT
    if (use_rtm) {
      Label L_regular_inflated_unlock;
      int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);

@@ -839,13 +858,15 @@
    cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)), 0);
    jccb(Assembler::equal, LNotRecursive);
  
    // Recursive inflated unlock
    decq(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
-   jmpb(LSuccess);
+   xorl(tmpReg, tmpReg); // Set ZF == 1
+   jmp(DONE_LABEL);
  
    bind(LNotRecursive);
+ 
    movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
    orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
    jccb  (Assembler::notZero, CheckSucc);
    // Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
    movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);

@@ -889,12 +910,13 @@
    // coherence traffic on the lock *and* artificially extended the critical section
    // length while by virtue of passing control into the slow path.
  
    // box is really RAX -- the following CMPXCHG depends on that binding
    // cmpxchg R,[M] is equivalent to rax = CAS(M,rax,R)
+   movptr(scrReg, Address(r15_thread, JavaThread::lock_id_offset()));
    lock();
-   cmpxchgptr(r15_thread, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
+   cmpxchgptr(scrReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
    // There's no successor so we tried to regrab the lock.
    // If that didn't work, then another thread grabbed the
    // lock so we're done (and exit was a success).
    jccb  (Assembler::notEqual, LSuccess);
    // Intentional fall-through into slow path

@@ -911,42 +933,35 @@
    if (LockingMode == LM_LEGACY) {
      bind  (Stacked);
      movptr(tmpReg, Address (boxReg, 0));      // re-fetch
      lock();
      cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
-     // Intentional fall-thru into DONE_LABEL
-   }
- 
-   bind(DONE_LABEL);
- 
-   // ZFlag == 1 count in fast path
-   // ZFlag == 0 count in slow path
-   jccb(Assembler::notZero, NO_COUNT);
- 
-   bind(COUNT);
-   // Count monitors in fast path
+     jccb(Assembler::notZero, DONE_LABEL);
+     // Count monitors in fast path
  #ifndef _LP64
-   get_thread(tmpReg);
-   decrementl(Address(tmpReg, JavaThread::held_monitor_count_offset()));
+     get_thread(tmpReg);
+     decrementl(Address(tmpReg, JavaThread::held_monitor_count_offset()));
  #else // _LP64
-   decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
+     decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
  #endif
+     xorl(tmpReg, tmpReg); // Set ZF == 1
+   }
  
-   xorl(tmpReg, tmpReg); // Set ZF == 1
- 
-   bind(NO_COUNT);
+   // ZFlag == 1 -> Success
+   // ZFlag == 0 -> Failure - force control through the slow path
+   bind(DONE_LABEL);
  }
  
  void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register rax_reg,
                                                Register t, Register thread) {
    assert(LockingMode == LM_LIGHTWEIGHT, "must be");
    assert(rax_reg == rax, "Used for CAS");
    assert_different_registers(obj, box, rax_reg, t, thread);
  
    // Handle inflated monitor.
    Label inflated;
-   // Finish fast lock successfully. ZF value is irrelevant.
+   // Finish fast lock successfully.
    Label locked;
    // Finish fast lock unsuccessfully. MUST jump with ZF == 0
    Label slow_path;
  
    if (DiagnoseSyncOnValueBasedClasses != 0) {

@@ -991,36 +1006,35 @@
  
      bind(push);
      // After successful lock, push object on lock-stack.
      movptr(Address(thread, top), obj);
      addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
+     xorl(rax_reg, rax_reg);
      jmpb(locked);
    }
  
    { // Handle inflated monitor.
      bind(inflated);
  
      const Register tagged_monitor = mark;
  
      // CAS owner (null => current thread).
      xorptr(rax_reg, rax_reg);
-     lock(); cmpxchgptr(thread, Address(tagged_monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
+     movptr(box, Address(thread, JavaThread::lock_id_offset()));
+     lock(); cmpxchgptr(box, Address(tagged_monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
      jccb(Assembler::equal, locked);
  
      // Check if recursive.
-     cmpptr(thread, rax_reg);
+     cmpptr(box, rax_reg);
      jccb(Assembler::notEqual, slow_path);
  
      // Recursive.
      increment(Address(tagged_monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
+     xorl(rax_reg, rax_reg);
    }
  
    bind(locked);
-   increment(Address(thread, JavaThread::held_monitor_count_offset()));
-   // Set ZF = 1
-   xorl(rax_reg, rax_reg);
- 
  #ifdef ASSERT
    // Check that locked label is reached with ZF set.
    Label zf_correct;
    jccb(Assembler::zero, zf_correct);
    stop("Fast Lock ZF != 1");

@@ -1034,31 +1048,28 @@
    bind(zf_correct);
  #endif
    // C2 uses the value of ZF to determine the continuation.
  }
  
- void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register reg_rax, Register t, Register thread) {
+ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register reg_rax, Register t1, Register t2, Register thread) {
    assert(LockingMode == LM_LIGHTWEIGHT, "must be");
    assert(reg_rax == rax, "Used for CAS");
-   assert_different_registers(obj, reg_rax, t);
+   assert_different_registers(obj, reg_rax, t1, t2);
  
    // Handle inflated monitor.
    Label inflated, inflated_check_lock_stack;
    // Finish fast unlock successfully.  MUST jump with ZF == 1
    Label unlocked;
  
-   // Assume success.
-   decrement(Address(thread, JavaThread::held_monitor_count_offset()));
- 
-   const Register mark = t;
+   const Register mark = t1;
    const Register top = reg_rax;
  
    Label dummy;
    C2FastUnlockLightweightStub* stub = nullptr;
  
    if (!Compile::current()->output()->in_scratch_emit_size()) {
-     stub = new (Compile::current()->comp_arena()) C2FastUnlockLightweightStub(obj, mark, reg_rax, thread);
+     stub = new (Compile::current()->comp_arena()) C2FastUnlockLightweightStub(obj, mark, reg_rax, t2, thread);
      Compile::current()->output()->add_stub(stub);
    }
  
    Label& push_and_slow_path = stub == nullptr ? dummy : stub->push_and_slow_path();
    Label& check_successor = stub == nullptr ? dummy : stub->check_successor();

@@ -1147,11 +1158,11 @@
      jmpb(unlocked);
  
      // Recursive unlock.
      bind(recursive);
      decrement(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
-     xorl(t, t);
+     xorl(t1, t1);
  #endif
    }
  
    bind(unlocked);
    if (stub != nullptr) {
< prev index next >