< prev index next >

src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp

Print this page
*** 24,13 ***
--- 24,15 ---
  
  #include "precompiled.hpp"
  #include "asm/assembler.hpp"
  #include "asm/assembler.inline.hpp"
  #include "oops/methodData.hpp"
+ #include "opto/c2_CodeStubs.hpp"
  #include "opto/c2_MacroAssembler.hpp"
  #include "opto/intrinsicnode.hpp"
  #include "opto/opcodes.hpp"
+ #include "opto/output.hpp"
  #include "opto/subnode.hpp"
  #include "runtime/biasedLocking.hpp"
  #include "runtime/objectMonitor.hpp"
  #include "runtime/stubRoutines.hpp"
  

*** 444,11 ***
  // obj: object to lock
  // box: on-stack box address (displaced header location) - KILLED
  // rax,: tmp -- KILLED
  // scr: tmp -- KILLED
  void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,
!                                  Register scrReg, Register cx1Reg, Register cx2Reg,
                                   BiasedLockingCounters* counters,
                                   RTMLockingCounters* rtm_counters,
                                   RTMLockingCounters* stack_rtm_counters,
                                   Metadata* method_data,
                                   bool use_rtm, bool profile_rtm) {
--- 446,11 ---
  // obj: object to lock
  // box: on-stack box address (displaced header location) - KILLED
  // rax,: tmp -- KILLED
  // scr: tmp -- KILLED
  void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,
!                                  Register scrReg, Register cx1Reg, Register cx2Reg, Register thread,
                                   BiasedLockingCounters* counters,
                                   RTMLockingCounters* rtm_counters,
                                   RTMLockingCounters* stack_rtm_counters,
                                   Metadata* method_data,
                                   bool use_rtm, bool profile_rtm) {

*** 513,31 ***
  
    movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));          // [FETCH]
    testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral|biased
    jccb(Assembler::notZero, IsInflated);
  
!   // Attempt stack-locking ...
!   orptr (tmpReg, markWord::unlocked_value);
!   movptr(Address(boxReg, 0), tmpReg);          // Anticipate successful CAS
!   lock();
!   cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes()));      // Updates tmpReg
!   if (counters != NULL) {
!     cond_inc32(Assembler::equal,
!                ExternalAddress((address)counters->fast_path_entry_count_addr()));
!   }
!   jcc(Assembler::equal, DONE_LABEL);           // Success
! 
!   // Recursive locking.
!   // The object is stack-locked: markword contains stack pointer to BasicLock.
!   // Locked by current thread if difference with current SP is less than one page.
!   subptr(tmpReg, rsp);
!   // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
!   andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
!   movptr(Address(boxReg, 0), tmpReg);
!   if (counters != NULL) {
!     cond_inc32(Assembler::equal,
!                ExternalAddress((address)counters->fast_path_entry_count_addr()));
    }
    jmp(DONE_LABEL);
  
    bind(IsInflated);
    // The object is inflated. tmpReg contains pointer to ObjectMonitor* + markWord::monitor_value
--- 515,49 ---
  
    movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));          // [FETCH]
    testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral|biased
    jccb(Assembler::notZero, IsInflated);
  
!   if (UseFastLocking) {
! #ifdef _LP64
!     fast_lock_impl(objReg, tmpReg, thread, scrReg, DONE_LABEL, false);
!     xorl(tmpReg, tmpReg); // Set ZF=1 to indicate success
! #else
!     // We can not emit the lock-stack-check in verified_entry() because we don't have enough
!     // registers (for thread ptr). Therefor we have to emit the lock-stack-check in
!     // fast_lock_impl(). However, that check can take a slow-path with ZF=1, therefore
!     // we need to handle it specially and force ZF=0 before taking the actual slow-path.
!     Label slow;
!     fast_lock_impl(objReg, tmpReg, thread, scrReg, slow);
!     xorl(tmpReg, tmpReg);
!     jmp(DONE_LABEL);
!     bind(slow);
!     testptr(objReg, objReg); // ZF=0 to indicate failure
! #endif
!   } else {
!     // Attempt stack-locking ...
!     orptr (tmpReg, markWord::unlocked_value);
!     movptr(Address(boxReg, 0), tmpReg);          // Anticipate successful CAS
!     lock();
+     cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes()));      // Updates tmpReg
+     if (counters != NULL) {
+       cond_inc32(Assembler::equal,
+                  ExternalAddress((address)counters->fast_path_entry_count_addr()));
+     }
+     jcc(Assembler::equal, DONE_LABEL);           // Success
+ 
+     // Recursive locking.
+     // The object is stack-locked: markword contains stack pointer to BasicLock.
+     // Locked by current thread if difference with current SP is less than one page.
+     subptr(tmpReg, rsp);
+     // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
+     andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
+     movptr(Address(boxReg, 0), tmpReg);
+     if (counters != NULL) {
+       cond_inc32(Assembler::equal,
+                  ExternalAddress((address)counters->fast_path_entry_count_addr()));
+     }
    }
    jmp(DONE_LABEL);
  
    bind(IsInflated);
    // The object is inflated. tmpReg contains pointer to ObjectMonitor* + markWord::monitor_value

*** 574,20 ***
    // rsp or the address of the box (in scr) into &m->owner.  If the CAS succeeds
    // we later store "Self" into m->Owner.  Transiently storing a stack address
    // (rsp or the address of the box) into  m->owner is harmless.
    // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
    lock();
!   cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
    movptr(Address(scrReg, 0), 3);          // box->_displaced_header = 3
-   // If we weren't able to swing _owner from NULL to the BasicLock
-   // then take the slow path.
-   jccb  (Assembler::notZero, DONE_LABEL);
-   // update _owner from BasicLock to thread
-   get_thread (scrReg);                    // beware: clobbers ICCs
-   movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg);
-   xorptr(boxReg, boxReg);                 // set icc.ZFlag = 1 to indicate success
- 
    // If the CAS fails we can either retry or pass control to the slow path.
    // We use the latter tactic.
    // Pass the CAS result in the icc.ZFlag into DONE_LABEL
    // If the CAS was successful ...
    //   Self has acquired the lock
--- 594,12 ---
    // rsp or the address of the box (in scr) into &m->owner.  If the CAS succeeds
    // we later store "Self" into m->Owner.  Transiently storing a stack address
    // (rsp or the address of the box) into  m->owner is harmless.
    // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
    lock();
!   cmpxchgptr(thread, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
    movptr(Address(scrReg, 0), 3);          // box->_displaced_header = 3
    // If the CAS fails we can either retry or pass control to the slow path.
    // We use the latter tactic.
    // Pass the CAS result in the icc.ZFlag into DONE_LABEL
    // If the CAS was successful ...
    //   Self has acquired the lock

*** 682,15 ***
      jmp(DONE_LABEL);                                                  // ... and we're done
      bind(L_regular_unlock);
    }
  #endif
  
!   cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD);                   // Examine the displaced header
!   jcc   (Assembler::zero, DONE_LABEL);                              // 0 indicates recursive stack-lock
    movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Examine the object's markword
    testptr(tmpReg, markWord::monitor_value);                         // Inflated?
!   jccb  (Assembler::zero, Stacked);
  
    // It's inflated.
  #if INCLUDE_RTM_OPT
    if (use_rtm) {
      Label L_regular_inflated_unlock;
--- 694,32 ---
      jmp(DONE_LABEL);                                                  // ... and we're done
      bind(L_regular_unlock);
    }
  #endif
  
!   if (!UseFastLocking) {
!     cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD);                   // Examine the displaced header
+     jcc   (Assembler::zero, DONE_LABEL);                              // 0 indicates recursive stack-lock
+   }
    movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Examine the object's markword
    testptr(tmpReg, markWord::monitor_value);                         // Inflated?
!   jcc(Assembler::zero, Stacked);
+ 
+   if (UseFastLocking) {
+     // If the owner is ANONYMOUS, we need to fix it.
+     testb(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int) (intptr_t) ANONYMOUS_OWNER);
+ #ifdef _LP64
+     C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmpReg);
+     Compile::current()->output()->add_stub(stub);
+     jcc(Assembler::notEqual, stub->entry());
+     bind(stub->continuation());
+ #else
+     // We can't easily implement this optimization on 32 bit because we don't have a thread register.
+     // Call the slow-path instead.
+     jcc(Assembler::notEqual, DONE_LABEL);
+ #endif
+   }
  
    // It's inflated.
  #if INCLUDE_RTM_OPT
    if (use_rtm) {
      Label L_regular_inflated_unlock;

*** 737,18 ***
    jccb  (Assembler::notZero, CheckSucc);
    movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
    jmpb  (DONE_LABEL);
  
    bind (Stacked);
!   // It's not inflated and it's not recursively stack-locked and it's not biased.
!   // It must be stack-locked.
!   // Try to reset the header to displaced header.
!   // The "box" value on the stack is stable, so we can reload
!   // and be assured we observe the same value as above.
!   movptr(tmpReg, Address(boxReg, 0));
!   lock();
!   cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
    // Intention fall-thru into DONE_LABEL
  
    // DONE_LABEL is a hot target - we'd really like to place it at the
    // start of cache line by padding with NOPs.
    // See the AMD and Intel software optimization manuals for the
--- 766,24 ---
    jccb  (Assembler::notZero, CheckSucc);
    movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
    jmpb  (DONE_LABEL);
  
    bind (Stacked);
!   if (UseFastLocking) {
!     mov(boxReg, tmpReg);
!     fast_unlock_impl(objReg, boxReg, tmpReg, DONE_LABEL);
!     xorl(tmpReg, tmpReg);
!   } else {
!     // It's not inflated and it's not recursively stack-locked and it's not biased.
!     // It must be stack-locked.
!     // Try to reset the header to displaced header.
+     // The "box" value on the stack is stable, so we can reload
+     // and be assured we observe the same value as above.
+     movptr(tmpReg, Address(boxReg, 0));
+     lock();
+     cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
+   }
    // Intention fall-thru into DONE_LABEL
  
    // DONE_LABEL is a hot target - we'd really like to place it at the
    // start of cache line by padding with NOPs.
    // See the AMD and Intel software optimization manuals for the

*** 829,13 ***
    bind  (LSuccess);
    testl (boxReg, 0);                      // set ICC.ZF=1 to indicate success
    jmpb  (DONE_LABEL);
  
    bind  (Stacked);
!   movptr(tmpReg, Address (boxReg, 0));      // re-fetch
!   lock();
!   cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
  
  #endif
    bind(DONE_LABEL);
  }
  
--- 864,20 ---
    bind  (LSuccess);
    testl (boxReg, 0);                      // set ICC.ZF=1 to indicate success
    jmpb  (DONE_LABEL);
  
    bind  (Stacked);
! 
!   if (UseFastLocking) {
!     mov(boxReg, tmpReg);
+     fast_unlock_impl(objReg, boxReg, tmpReg, DONE_LABEL);
+     xorl(tmpReg, tmpReg);
+   } else {
+     movptr(tmpReg, Address (boxReg, 0));      // re-fetch
+     lock();
+     cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
+   }
  
  #endif
    bind(DONE_LABEL);
  }
  
< prev index next >