< prev index next > src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
Print this page
// Finish fast lock successfully. ZF value is irrelevant.
Label locked;
// Finish fast lock unsuccessfully. MUST jump with ZF == 0
Label slow_path;
+ if (UseObjectMonitorTable) {
+ // Clear cache in case fast locking succeeds.
+ movptr(Address(box, BasicLock::object_monitor_cache_offset_in_bytes()), 0);
+ }
+
if (DiagnoseSyncOnValueBasedClasses != 0) {
load_klass(rax_reg, obj, t);
movl(rax_reg, Address(rax_reg, Klass::access_flags_offset()));
testl(rax_reg, JVM_ACC_IS_VALUE_BASED_CLASS);
jcc(Assembler::notZero, slow_path);
{ // Lightweight Lock
Label push;
! const Register top = box;
// Load the mark.
movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
// Prefetch top.
{ // Lightweight Lock
Label push;
! const Register top = UseObjectMonitorTable ? rax_reg : box;
// Load the mark.
movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
// Prefetch top.
orptr(rax_reg, markWord::unlocked_value);
andptr(mark, ~(int32_t)markWord::unlocked_value);
lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
jcc(Assembler::notEqual, slow_path);
bind(push);
// After successful lock, push object on lock-stack.
movptr(Address(thread, top), obj);
addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
jmpb(locked);
}
{ // Handle inflated monitor.
bind(inflated);
! const Register tagged_monitor = mark;
// CAS owner (null => current thread).
xorptr(rax_reg, rax_reg);
! lock(); cmpxchgptr(thread, Address(tagged_monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
! jccb(Assembler::equal, locked);
// Check if recursive.
cmpptr(thread, rax_reg);
jccb(Assembler::notEqual, slow_path);
// Recursive.
! increment(Address(tagged_monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
}
bind(locked);
increment(Address(thread, JavaThread::held_monitor_count_offset()));
// Set ZF = 1
orptr(rax_reg, markWord::unlocked_value);
andptr(mark, ~(int32_t)markWord::unlocked_value);
lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
jcc(Assembler::notEqual, slow_path);
+ if (UseObjectMonitorTable) {
+ // Need to reload top, clobbered by CAS.
+ movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
+ }
bind(push);
// After successful lock, push object on lock-stack.
movptr(Address(thread, top), obj);
addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
jmpb(locked);
}
{ // Handle inflated monitor.
bind(inflated);
! const Register monitor = t;
+
+ if (!UseObjectMonitorTable) {
+ assert(mark == monitor, "should be the same here");
+ } else {
+ // Uses ObjectMonitorTable. Look for the monitor in the om_cache.
+ // Fetch ObjectMonitor* from the cache or take the slow-path.
+ Label monitor_found;
+
+ // Load cache address
+ lea(t, Address(thread, JavaThread::om_cache_oops_offset()));
+
+ const int num_unrolled = 2;
+ for (int i = 0; i < num_unrolled; i++) {
+ cmpptr(obj, Address(t));
+ jccb(Assembler::equal, monitor_found);
+ if (i + 1 != num_unrolled) {
+ increment(t, in_bytes(OMCache::oop_to_oop_difference()));
+ }
+ }
+
+ // Loop after unrolling, advance iterator.
+ increment(t, in_bytes(OMCache::oop_to_oop_difference()));
+
+ Label loop;
+
+ // Search for obj in cache.
+ bind(loop);
+
+ // Check for match.
+ cmpptr(obj, Address(t));
+ jccb(Assembler::equal, monitor_found);
+
+ // Search until null encountered, guaranteed _null_sentinel at end.
+ cmpptr(Address(t), 1);
+ jcc(Assembler::below, slow_path); // 0 check, but with ZF=0 when *t == 0
+ increment(t, in_bytes(OMCache::oop_to_oop_difference()));
+ jmpb(loop);
+
+ // Cache hit.
+ bind(monitor_found);
+ movptr(monitor, Address(t, OMCache::oop_to_monitor_difference()));
+ }
+ const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast<int>(markWord::monitor_value));
+ const Address recursions_address{monitor, ObjectMonitor::recursions_offset() - monitor_tag};
+ const Address owner_address{monitor, ObjectMonitor::owner_offset() - monitor_tag};
+
+ Label monitor_locked;
+ // Lock the monitor.
// CAS owner (null => current thread).
xorptr(rax_reg, rax_reg);
! lock(); cmpxchgptr(thread, owner_address);
! jccb(Assembler::equal, monitor_locked);
// Check if recursive.
cmpptr(thread, rax_reg);
jccb(Assembler::notEqual, slow_path);
// Recursive.
! increment(recursions_address);
+
+ bind(monitor_locked);
+ if (UseObjectMonitorTable) {
+ // Cache the monitor for unlock
+ movptr(Address(box, BasicLock::object_monitor_cache_offset_in_bytes()), monitor);
+ }
}
bind(locked);
increment(Address(thread, JavaThread::held_monitor_count_offset()));
// Set ZF = 1
// Assume success.
decrement(Address(thread, JavaThread::held_monitor_count_offset()));
const Register mark = t;
! const Register top = reg_rax;
Label dummy;
C2FastUnlockLightweightStub* stub = nullptr;
if (!Compile::current()->output()->in_scratch_emit_size()) {
// Assume success.
decrement(Address(thread, JavaThread::held_monitor_count_offset()));
const Register mark = t;
! const Register monitor = t;
+ const Register top = UseObjectMonitorTable ? t : reg_rax;
+ const Register box = reg_rax;
Label dummy;
C2FastUnlockLightweightStub* stub = nullptr;
if (!Compile::current()->output()->in_scratch_emit_size()) {
Compile::current()->output()->add_stub(stub);
}
Label& push_and_slow_path = stub == nullptr ? dummy : stub->push_and_slow_path();
Label& check_successor = stub == nullptr ? dummy : stub->check_successor();
{ // Lightweight Unlock
// Load top.
movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
! // Prefetch mark.
! movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
// Check if obj is top of lock-stack.
cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
// Top of lock stack was not obj. Must be monitor.
jcc(Assembler::notEqual, inflated_check_lock_stack);
Compile::current()->output()->add_stub(stub);
}
Label& push_and_slow_path = stub == nullptr ? dummy : stub->push_and_slow_path();
Label& check_successor = stub == nullptr ? dummy : stub->check_successor();
+ Label& slow_path = stub == nullptr ? dummy : stub->slow_path();
{ // Lightweight Unlock
// Load top.
movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
! if (!UseObjectMonitorTable) {
! // Prefetch mark.
+ movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+ }
// Check if obj is top of lock-stack.
cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
// Top of lock stack was not obj. Must be monitor.
jcc(Assembler::notEqual, inflated_check_lock_stack);
cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
jcc(Assembler::equal, unlocked);
// We elide the monitor check, let the CAS fail instead.
+ if (UseObjectMonitorTable) {
+ // Load mark.
+ movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+ }
+
// Try to unlock. Transition lock bits 0b00 => 0b01
movptr(reg_rax, mark);
andptr(reg_rax, ~(int32_t)markWord::lock_mask);
orptr(mark, markWord::unlocked_value);
lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
jcc(Assembler::below, check_done);
cmpptr(obj, Address(thread, top));
jccb(Assembler::notEqual, inflated_check_lock_stack);
stop("Fast Unlock lock on stack");
bind(check_done);
testptr(mark, markWord::monitor_value);
jccb(Assembler::notZero, inflated);
stop("Fast Unlock not monitor");
#endif
bind(inflated);
! // mark contains the tagged ObjectMonitor*.
! const Register monitor = mark;
!
! #ifndef _LP64
! // Check if recursive.
! xorptr(reg_rax, reg_rax);
! orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
! jcc(Assembler::notZero, check_successor);
!
! // Check if the entry lists are empty.
! movptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
! orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
! jcc(Assembler::notZero, check_successor);
- // Release lock.
- movptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
- #else // _LP64
Label recursive;
// Check if recursive.
! cmpptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)), 0);
jccb(Assembler::notEqual, recursive);
// Check if the entry lists are empty.
! movptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
! orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
jcc(Assembler::notZero, check_successor);
// Release lock.
! movptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
jmpb(unlocked);
// Recursive unlock.
bind(recursive);
! decrement(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
xorl(t, t);
- #endif
}
bind(unlocked);
if (stub != nullptr) {
bind(stub->unlocked_continuation());
jcc(Assembler::below, check_done);
cmpptr(obj, Address(thread, top));
jccb(Assembler::notEqual, inflated_check_lock_stack);
stop("Fast Unlock lock on stack");
bind(check_done);
+ if (UseObjectMonitorTable) {
+ movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+ }
testptr(mark, markWord::monitor_value);
jccb(Assembler::notZero, inflated);
stop("Fast Unlock not monitor");
#endif
bind(inflated);
! if (!UseObjectMonitorTable) {
! assert(mark == monitor, "should be the same here");
! } else {
! // Uses ObjectMonitorTable. Look for the monitor in our BasicLock on the stack.
! movptr(monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
! // null check with ZF == 0, no valid pointer below alignof(ObjectMonitor*)
! cmpptr(monitor, alignof(ObjectMonitor*));
! jcc(Assembler::below, slow_path);
! }
! const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast<int>(markWord::monitor_value));
! const Address recursions_address{monitor, ObjectMonitor::recursions_offset() - monitor_tag};
! const Address cxq_address{monitor, ObjectMonitor::cxq_offset() - monitor_tag};
! const Address EntryList_address{monitor, ObjectMonitor::EntryList_offset() - monitor_tag};
+ const Address owner_address{monitor, ObjectMonitor::owner_offset() - monitor_tag};
Label recursive;
// Check if recursive.
! cmpptr(recursions_address, 0);
jccb(Assembler::notEqual, recursive);
// Check if the entry lists are empty.
! movptr(reg_rax, cxq_address);
! orptr(reg_rax, EntryList_address);
jcc(Assembler::notZero, check_successor);
// Release lock.
! movptr(owner_address, NULL_WORD);
jmpb(unlocked);
// Recursive unlock.
bind(recursive);
! decrement(recursions_address);
xorl(t, t);
}
bind(unlocked);
if (stub != nullptr) {
bind(stub->unlocked_continuation());
} else {
assert(bt == T_FLOAT, "");
vpermps(dst, shuffle, src, vlen_enc);
}
}
+
+ #ifdef _LP64
+ void C2_MacroAssembler::load_nklass_compact_c2(Register dst, Register obj, Register index, Address::ScaleFactor scale, int disp) {
+ // Note: Don't clobber obj anywhere in that method!
+
+ // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
+ // obj-start, so that we can load from the object's mark-word instead. Usually the address
+ // comes as obj-start in obj and klass_offset_in_bytes in disp. However, sometimes C2
+ // emits code that pre-computes obj-start + klass_offset_in_bytes into a register, and
+ // then passes that register as obj and 0 in disp. The following code extracts the base
+ // and offset to load the mark-word.
+ int offset = oopDesc::mark_offset_in_bytes() + disp - oopDesc::klass_offset_in_bytes();
+ movq(dst, Address(obj, index, scale, offset));
+ shrq(dst, markWord::klass_shift);
+ }
+ #endif
< prev index next >