< prev index next > src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
Print this page
#include "precompiled.hpp"
#include "asm/assembler.hpp"
#include "asm/assembler.inline.hpp"
#include "oops/methodData.hpp"
+ #include "opto/c2_CodeStubs.hpp"
#include "opto/c2_MacroAssembler.hpp"
#include "opto/intrinsicnode.hpp"
#include "opto/opcodes.hpp"
+ #include "opto/output.hpp"
#include "opto/subnode.hpp"
#include "runtime/biasedLocking.hpp"
#include "runtime/objectMonitor.hpp"
#include "runtime/stubRoutines.hpp"
// obj: object to lock
// box: on-stack box address (displaced header location) - KILLED
// rax,: tmp -- KILLED
// scr: tmp -- KILLED
void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,
! Register scrReg, Register cx1Reg, Register cx2Reg,
BiasedLockingCounters* counters,
RTMLockingCounters* rtm_counters,
RTMLockingCounters* stack_rtm_counters,
Metadata* method_data,
bool use_rtm, bool profile_rtm) {
// obj: object to lock
// box: on-stack box address (displaced header location) - KILLED
// rax,: tmp -- KILLED
// scr: tmp -- KILLED
void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,
! Register scrReg, Register cx1Reg, Register cx2Reg, Register thread,
BiasedLockingCounters* counters,
RTMLockingCounters* rtm_counters,
RTMLockingCounters* stack_rtm_counters,
Metadata* method_data,
bool use_rtm, bool profile_rtm) {
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // [FETCH]
testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral|biased
jccb(Assembler::notZero, IsInflated);
! // Attempt stack-locking ...
! orptr (tmpReg, markWord::unlocked_value);
! movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
! lock();
! cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Updates tmpReg
! if (counters != NULL) {
! cond_inc32(Assembler::equal,
! ExternalAddress((address)counters->fast_path_entry_count_addr()));
! }
! jcc(Assembler::equal, DONE_LABEL); // Success
!
! // Recursive locking.
! // The object is stack-locked: markword contains stack pointer to BasicLock.
! // Locked by current thread if difference with current SP is less than one page.
! subptr(tmpReg, rsp);
! // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
! andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
! movptr(Address(boxReg, 0), tmpReg);
! if (counters != NULL) {
! cond_inc32(Assembler::equal,
! ExternalAddress((address)counters->fast_path_entry_count_addr()));
}
jmp(DONE_LABEL);
bind(IsInflated);
// The object is inflated. tmpReg contains pointer to ObjectMonitor* + markWord::monitor_value
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // [FETCH]
testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral|biased
jccb(Assembler::notZero, IsInflated);
! if (UseFastLocking) {
! #ifdef _LP64
! fast_lock_impl(objReg, tmpReg, thread, scrReg, DONE_LABEL, false);
! xorl(tmpReg, tmpReg); // Set ZF=1 to indicate success
! #else
! // We can not emit the lock-stack-check in verified_entry() because we don't have enough
! // registers (for thread ptr). Therefor we have to emit the lock-stack-check in
! // fast_lock_impl(). However, that check can take a slow-path with ZF=1, therefore
! // we need to handle it specially and force ZF=0 before taking the actual slow-path.
! Label slow;
! fast_lock_impl(objReg, tmpReg, thread, scrReg, slow);
! xorl(tmpReg, tmpReg);
! jmp(DONE_LABEL);
! bind(slow);
! testptr(objReg, objReg); // ZF=0 to indicate failure
! #endif
! } else {
! // Attempt stack-locking ...
! orptr (tmpReg, markWord::unlocked_value);
! movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
! lock();
+ cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Updates tmpReg
+ if (counters != NULL) {
+ cond_inc32(Assembler::equal,
+ ExternalAddress((address)counters->fast_path_entry_count_addr()));
+ }
+ jcc(Assembler::equal, DONE_LABEL); // Success
+
+ // Recursive locking.
+ // The object is stack-locked: markword contains stack pointer to BasicLock.
+ // Locked by current thread if difference with current SP is less than one page.
+ subptr(tmpReg, rsp);
+ // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
+ andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
+ movptr(Address(boxReg, 0), tmpReg);
+ if (counters != NULL) {
+ cond_inc32(Assembler::equal,
+ ExternalAddress((address)counters->fast_path_entry_count_addr()));
+ }
}
jmp(DONE_LABEL);
bind(IsInflated);
// The object is inflated. tmpReg contains pointer to ObjectMonitor* + markWord::monitor_value
// rsp or the address of the box (in scr) into &m->owner. If the CAS succeeds
// we later store "Self" into m->Owner. Transiently storing a stack address
// (rsp or the address of the box) into m->owner is harmless.
// Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand.
lock();
! cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
movptr(Address(scrReg, 0), 3); // box->_displaced_header = 3
- // If we weren't able to swing _owner from NULL to the BasicLock
- // then take the slow path.
- jccb (Assembler::notZero, DONE_LABEL);
- // update _owner from BasicLock to thread
- get_thread (scrReg); // beware: clobbers ICCs
- movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg);
- xorptr(boxReg, boxReg); // set icc.ZFlag = 1 to indicate success
-
// If the CAS fails we can either retry or pass control to the slow path.
// We use the latter tactic.
// Pass the CAS result in the icc.ZFlag into DONE_LABEL
// If the CAS was successful ...
// Self has acquired the lock
// rsp or the address of the box (in scr) into &m->owner. If the CAS succeeds
// we later store "Self" into m->Owner. Transiently storing a stack address
// (rsp or the address of the box) into m->owner is harmless.
// Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand.
lock();
! cmpxchgptr(thread, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
movptr(Address(scrReg, 0), 3); // box->_displaced_header = 3
// If the CAS fails we can either retry or pass control to the slow path.
// We use the latter tactic.
// Pass the CAS result in the icc.ZFlag into DONE_LABEL
// If the CAS was successful ...
// Self has acquired the lock
jmp(DONE_LABEL); // ... and we're done
bind(L_regular_unlock);
}
#endif
! cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
! jcc (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Examine the object's markword
testptr(tmpReg, markWord::monitor_value); // Inflated?
! jccb (Assembler::zero, Stacked);
// It's inflated.
#if INCLUDE_RTM_OPT
if (use_rtm) {
Label L_regular_inflated_unlock;
jmp(DONE_LABEL); // ... and we're done
bind(L_regular_unlock);
}
#endif
! if (!UseFastLocking) {
! cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
+ jcc (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock
+ }
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Examine the object's markword
testptr(tmpReg, markWord::monitor_value); // Inflated?
! jcc(Assembler::zero, Stacked);
+
+ if (UseFastLocking) {
+ // If the owner is ANONYMOUS, we need to fix it.
+ testb(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int) (intptr_t) ANONYMOUS_OWNER);
+ #ifdef _LP64
+ C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmpReg);
+ Compile::current()->output()->add_stub(stub);
+ jcc(Assembler::notEqual, stub->entry());
+ bind(stub->continuation());
+ #else
+ // We can't easily implement this optimization on 32 bit because we don't have a thread register.
+ // Call the slow-path instead.
+ jcc(Assembler::notEqual, DONE_LABEL);
+ #endif
+ }
// It's inflated.
#if INCLUDE_RTM_OPT
if (use_rtm) {
Label L_regular_inflated_unlock;
jccb (Assembler::notZero, CheckSucc);
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
jmpb (DONE_LABEL);
bind (Stacked);
! // It's not inflated and it's not recursively stack-locked and it's not biased.
! // It must be stack-locked.
! // Try to reset the header to displaced header.
! // The "box" value on the stack is stable, so we can reload
! // and be assured we observe the same value as above.
! movptr(tmpReg, Address(boxReg, 0));
! lock();
! cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
// Intention fall-thru into DONE_LABEL
// DONE_LABEL is a hot target - we'd really like to place it at the
// start of cache line by padding with NOPs.
// See the AMD and Intel software optimization manuals for the
jccb (Assembler::notZero, CheckSucc);
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
jmpb (DONE_LABEL);
bind (Stacked);
! if (UseFastLocking) {
! mov(boxReg, tmpReg);
! fast_unlock_impl(objReg, boxReg, tmpReg, DONE_LABEL);
! xorl(tmpReg, tmpReg);
! } else {
! // It's not inflated and it's not recursively stack-locked and it's not biased.
! // It must be stack-locked.
! // Try to reset the header to displaced header.
+ // The "box" value on the stack is stable, so we can reload
+ // and be assured we observe the same value as above.
+ movptr(tmpReg, Address(boxReg, 0));
+ lock();
+ cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
+ }
// Intention fall-thru into DONE_LABEL
// DONE_LABEL is a hot target - we'd really like to place it at the
// start of cache line by padding with NOPs.
// See the AMD and Intel software optimization manuals for the
bind (LSuccess);
testl (boxReg, 0); // set ICC.ZF=1 to indicate success
jmpb (DONE_LABEL);
bind (Stacked);
! movptr(tmpReg, Address (boxReg, 0)); // re-fetch
! lock();
! cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
#endif
bind(DONE_LABEL);
}
bind (LSuccess);
testl (boxReg, 0); // set ICC.ZF=1 to indicate success
jmpb (DONE_LABEL);
bind (Stacked);
!
! if (UseFastLocking) {
! mov(boxReg, tmpReg);
+ fast_unlock_impl(objReg, boxReg, tmpReg, DONE_LABEL);
+ xorl(tmpReg, tmpReg);
+ } else {
+ movptr(tmpReg, Address (boxReg, 0)); // re-fetch
+ lock();
+ cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
+ }
#endif
bind(DONE_LABEL);
}
< prev index next >