< prev index next > src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
Print this page
#include "precompiled.hpp"
#include "asm/assembler.hpp"
#include "asm/assembler.inline.hpp"
#include "oops/methodData.hpp"
+ #include "opto/c2_CodeStubs.hpp"
#include "opto/c2_MacroAssembler.hpp"
#include "opto/intrinsicnode.hpp"
#include "opto/opcodes.hpp"
+ #include "opto/output.hpp"
#include "opto/subnode.hpp"
#include "runtime/biasedLocking.hpp"
#include "runtime/objectMonitor.hpp"
#include "runtime/stubRoutines.hpp"
// obj: object to lock
// box: on-stack box address (displaced header location) - KILLED
// rax,: tmp -- KILLED
// scr: tmp -- KILLED
void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,
! Register scrReg, Register cx1Reg, Register cx2Reg,
BiasedLockingCounters* counters,
RTMLockingCounters* rtm_counters,
RTMLockingCounters* stack_rtm_counters,
Metadata* method_data,
bool use_rtm, bool profile_rtm) {
// obj: object to lock
// box: on-stack box address (displaced header location) - KILLED
// rax,: tmp -- KILLED
// scr: tmp -- KILLED
void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,
! Register scrReg, Register cx1Reg, Register cx2Reg, Register thread,
BiasedLockingCounters* counters,
RTMLockingCounters* rtm_counters,
RTMLockingCounters* stack_rtm_counters,
Metadata* method_data,
bool use_rtm, bool profile_rtm) {
}
#endif // INCLUDE_RTM_OPT
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // [FETCH]
testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral|biased
! jccb(Assembler::notZero, IsInflated);
! // Attempt stack-locking ...
! orptr (tmpReg, markWord::unlocked_value);
! movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
! lock();
! cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Updates tmpReg
! if (counters != NULL) {
! cond_inc32(Assembler::equal,
! ExternalAddress((address)counters->fast_path_entry_count_addr()));
! }
! jcc(Assembler::equal, DONE_LABEL); // Success
!
! // Recursive locking.
! // The object is stack-locked: markword contains stack pointer to BasicLock.
! // Locked by current thread if difference with current SP is less than one page.
! subptr(tmpReg, rsp);
! // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
! andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
! movptr(Address(boxReg, 0), tmpReg);
! if (counters != NULL) {
! cond_inc32(Assembler::equal,
! ExternalAddress((address)counters->fast_path_entry_count_addr()));
}
jmp(DONE_LABEL);
bind(IsInflated);
// The object is inflated. tmpReg contains pointer to ObjectMonitor* + markWord::monitor_value
}
#endif // INCLUDE_RTM_OPT
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // [FETCH]
testptr(tmpReg, markWord::monitor_value); // inflated vs stack-locked|neutral|biased
! jcc(Assembler::notZero, IsInflated);
! if (LockingMode == LM_MONITOR) {
! // Clear ZF so that we take the slow path at the DONE label. objReg is known to be not 0.
! testptr(objReg, objReg);
! } else if (LockingMode == LM_LEGACY) {
! // Attempt stack-locking ...
! orptr (tmpReg, markWord::unlocked_value);
! movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
! lock();
! cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Updates tmpReg
! if (counters != NULL) {
! cond_inc32(Assembler::equal,
! ExternalAddress((address)counters->fast_path_entry_count_addr()));
! }
! jcc(Assembler::equal, DONE_LABEL); // Success
!
! // Recursive locking.
! // The object is stack-locked: markword contains stack pointer to BasicLock.
! // Locked by current thread if difference with current SP is less than one page.
! subptr(tmpReg, rsp);
! // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
! andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
+ movptr(Address(boxReg, 0), tmpReg);
+ if (counters != NULL) {
+ cond_inc32(Assembler::equal,
+ ExternalAddress((address)counters->fast_path_entry_count_addr()));
+ }
+ } else {
+ assert(LockingMode == LM_LIGHTWEIGHT, "");
+ fast_lock_impl(objReg, tmpReg, thread, scrReg, DONE_LABEL);
+ xorl(tmpReg, tmpReg); // Set ZF=1 to indicate success
}
jmp(DONE_LABEL);
bind(IsInflated);
// The object is inflated. tmpReg contains pointer to ObjectMonitor* + markWord::monitor_value
jmp(DONE_LABEL); // ... and we're done
bind(L_regular_unlock);
}
#endif
! cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
! jcc (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Examine the object's markword
! testptr(tmpReg, markWord::monitor_value); // Inflated?
! jccb (Assembler::zero, Stacked);
// It's inflated.
#if INCLUDE_RTM_OPT
if (use_rtm) {
Label L_regular_inflated_unlock;
int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
movptr(boxReg, Address(tmpReg, owner_offset));
testptr(boxReg, boxReg);
jccb(Assembler::notZero, L_regular_inflated_unlock);
xend();
! jmpb(DONE_LABEL);
bind(L_regular_inflated_unlock);
}
#endif
// Despite our balanced locking property we still check that m->_owner == Self
jmp(DONE_LABEL); // ... and we're done
bind(L_regular_unlock);
}
#endif
! if (LockingMode == LM_LEGACY) {
! cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
+ jcc (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock
+ }
movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Examine the object's markword
! if (LockingMode != LM_MONITOR) {
! testptr(tmpReg, markWord::monitor_value); // Inflated?
+ jcc(Assembler::zero, Stacked);
+ }
// It's inflated.
+ if (LockingMode == LM_LIGHTWEIGHT) {
+ // If the owner is ANONYMOUS, we need to fix it.
+ testb(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t) ObjectMonitor::ANONYMOUS_OWNER);
+ #ifdef _LP64
+ if (!Compile::current()->output()->in_scratch_emit_size()) {
+ C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmpReg, boxReg);
+ Compile::current()->output()->add_stub(stub);
+ jcc(Assembler::notEqual, stub->entry());
+ bind(stub->continuation());
+ } else
+ #endif
+ {
+ // We can't easily implement this optimization on 32 bit because we don't have a thread register.
+ // Call the slow-path instead.
+ jcc(Assembler::notEqual, DONE_LABEL);
+ }
+ }
+
#if INCLUDE_RTM_OPT
if (use_rtm) {
Label L_regular_inflated_unlock;
int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
movptr(boxReg, Address(tmpReg, owner_offset));
testptr(boxReg, boxReg);
jccb(Assembler::notZero, L_regular_inflated_unlock);
xend();
! jmp(DONE_LABEL);
bind(L_regular_inflated_unlock);
}
#endif
// Despite our balanced locking property we still check that m->_owner == Self
xorptr(boxReg, boxReg);
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
jccb (Assembler::notZero, DONE_LABEL);
movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
! jccb (Assembler::notZero, CheckSucc);
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
jmpb (DONE_LABEL);
- bind (Stacked);
- // It's not inflated and it's not recursively stack-locked and it's not biased.
- // It must be stack-locked.
- // Try to reset the header to displaced header.
- // The "box" value on the stack is stable, so we can reload
- // and be assured we observe the same value as above.
- movptr(tmpReg, Address(boxReg, 0));
- lock();
- cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
// Intention fall-thru into DONE_LABEL
// DONE_LABEL is a hot target - we'd really like to place it at the
// start of cache line by padding with NOPs.
// See the AMD and Intel software optimization manuals for the
xorptr(boxReg, boxReg);
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
jccb (Assembler::notZero, DONE_LABEL);
movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
! jccb (Assembler::notZero, DONE_LABEL);
movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
jmpb (DONE_LABEL);
// Intention fall-thru into DONE_LABEL
// DONE_LABEL is a hot target - we'd really like to place it at the
// start of cache line by padding with NOPs.
// See the AMD and Intel software optimization manuals for the
bind (LSuccess);
testl (boxReg, 0); // set ICC.ZF=1 to indicate success
jmpb (DONE_LABEL);
- bind (Stacked);
- movptr(tmpReg, Address (boxReg, 0)); // re-fetch
- lock();
- cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
-
#endif
bind(DONE_LABEL);
}
//-------------------------------------------------------------------------------------------
// Generic instructions support for use in .ad files C2 code generation
bind (LSuccess);
testl (boxReg, 0); // set ICC.ZF=1 to indicate success
jmpb (DONE_LABEL);
#endif
+ if (LockingMode != LM_MONITOR) {
+ bind (Stacked);
+ if (LockingMode == LM_LIGHTWEIGHT) {
+ mov(boxReg, tmpReg);
+ fast_unlock_impl(objReg, boxReg, tmpReg, DONE_LABEL);
+ xorl(tmpReg, tmpReg);
+ } else if (LockingMode == LM_LEGACY) {
+ movptr(tmpReg, Address (boxReg, 0)); // re-fetch
+ lock();
+ cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
+ }
+ }
bind(DONE_LABEL);
}
//-------------------------------------------------------------------------------------------
// Generic instructions support for use in .ad files C2 code generation
< prev index next >